1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/coproc.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Authors: Rusty Russell <rusty@rustcorp.com.au>
9 * Christoffer Dall <c.dall@virtualopensystems.com>
10 */
11
12 #include <linux/bitfield.h>
13 #include <linux/bsearch.h>
14 #include <linux/cacheinfo.h>
15 #include <linux/kvm_host.h>
16 #include <linux/mm.h>
17 #include <linux/printk.h>
18 #include <linux/uaccess.h>
19
20 #include <asm/cacheflush.h>
21 #include <asm/cputype.h>
22 #include <asm/debug-monitors.h>
23 #include <asm/esr.h>
24 #include <asm/kvm_arm.h>
25 #include <asm/kvm_emulate.h>
26 #include <asm/kvm_hyp.h>
27 #include <asm/kvm_mmu.h>
28 #include <asm/kvm_nested.h>
29 #include <asm/perf_event.h>
30 #include <asm/sysreg.h>
31
32 #include <trace/events/kvm.h>
33
34 #include "sys_regs.h"
35
36 #include "trace.h"
37
38 /*
39 * For AArch32, we only take care of what is being trapped. Anything
40 * that has to do with init and userspace access has to go via the
41 * 64bit interface.
42 */
43
44 static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
45 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
46 u64 val);
47
bad_trap(struct kvm_vcpu * vcpu,struct sys_reg_params * params,const struct sys_reg_desc * r,const char * msg)48 static bool bad_trap(struct kvm_vcpu *vcpu,
49 struct sys_reg_params *params,
50 const struct sys_reg_desc *r,
51 const char *msg)
52 {
53 WARN_ONCE(1, "Unexpected %s\n", msg);
54 print_sys_reg_instr(params);
55 kvm_inject_undefined(vcpu);
56 return false;
57 }
58
read_from_write_only(struct kvm_vcpu * vcpu,struct sys_reg_params * params,const struct sys_reg_desc * r)59 static bool read_from_write_only(struct kvm_vcpu *vcpu,
60 struct sys_reg_params *params,
61 const struct sys_reg_desc *r)
62 {
63 return bad_trap(vcpu, params, r,
64 "sys_reg read to write-only register");
65 }
66
write_to_read_only(struct kvm_vcpu * vcpu,struct sys_reg_params * params,const struct sys_reg_desc * r)67 static bool write_to_read_only(struct kvm_vcpu *vcpu,
68 struct sys_reg_params *params,
69 const struct sys_reg_desc *r)
70 {
71 return bad_trap(vcpu, params, r,
72 "sys_reg write to read-only register");
73 }
74
75 #define PURE_EL2_SYSREG(el2) \
76 case el2: { \
77 *el1r = el2; \
78 return true; \
79 }
80
81 #define MAPPED_EL2_SYSREG(el2, el1, fn) \
82 case el2: { \
83 *xlate = fn; \
84 *el1r = el1; \
85 return true; \
86 }
87
get_el2_to_el1_mapping(unsigned int reg,unsigned int * el1r,u64 (** xlate)(u64))88 static bool get_el2_to_el1_mapping(unsigned int reg,
89 unsigned int *el1r, u64 (**xlate)(u64))
90 {
91 switch (reg) {
92 PURE_EL2_SYSREG( VPIDR_EL2 );
93 PURE_EL2_SYSREG( VMPIDR_EL2 );
94 PURE_EL2_SYSREG( ACTLR_EL2 );
95 PURE_EL2_SYSREG( HCR_EL2 );
96 PURE_EL2_SYSREG( MDCR_EL2 );
97 PURE_EL2_SYSREG( HSTR_EL2 );
98 PURE_EL2_SYSREG( HACR_EL2 );
99 PURE_EL2_SYSREG( VTTBR_EL2 );
100 PURE_EL2_SYSREG( VTCR_EL2 );
101 PURE_EL2_SYSREG( RVBAR_EL2 );
102 PURE_EL2_SYSREG( TPIDR_EL2 );
103 PURE_EL2_SYSREG( HPFAR_EL2 );
104 PURE_EL2_SYSREG( CNTHCTL_EL2 );
105 MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
106 translate_sctlr_el2_to_sctlr_el1 );
107 MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
108 translate_cptr_el2_to_cpacr_el1 );
109 MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1,
110 translate_ttbr0_el2_to_ttbr0_el1 );
111 MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL );
112 MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1,
113 translate_tcr_el2_to_tcr_el1 );
114 MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1, NULL );
115 MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1, NULL );
116 MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1, NULL );
117 MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1, NULL );
118 MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1, NULL );
119 MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1, NULL );
120 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
121 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
122 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
123 default:
124 return false;
125 }
126 }
127
vcpu_read_sys_reg(const struct kvm_vcpu * vcpu,int reg)128 u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
129 {
130 u64 val = 0x8badf00d8badf00d;
131 u64 (*xlate)(u64) = NULL;
132 unsigned int el1r;
133
134 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
135 goto memory_read;
136
137 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
138 if (!is_hyp_ctxt(vcpu))
139 goto memory_read;
140
141 /*
142 * If this register does not have an EL1 counterpart,
143 * then read the stored EL2 version.
144 */
145 if (reg == el1r)
146 goto memory_read;
147
148 /*
149 * If we have a non-VHE guest and that the sysreg
150 * requires translation to be used at EL1, use the
151 * in-memory copy instead.
152 */
153 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
154 goto memory_read;
155
156 /* Get the current version of the EL1 counterpart. */
157 WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val));
158 return val;
159 }
160
161 /* EL1 register can't be on the CPU if the guest is in vEL2. */
162 if (unlikely(is_hyp_ctxt(vcpu)))
163 goto memory_read;
164
165 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
166 return val;
167
168 memory_read:
169 return __vcpu_sys_reg(vcpu, reg);
170 }
171
vcpu_write_sys_reg(struct kvm_vcpu * vcpu,u64 val,int reg)172 void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
173 {
174 u64 (*xlate)(u64) = NULL;
175 unsigned int el1r;
176
177 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
178 goto memory_write;
179
180 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
181 if (!is_hyp_ctxt(vcpu))
182 goto memory_write;
183
184 /*
185 * Always store a copy of the write to memory to avoid having
186 * to reverse-translate virtual EL2 system registers for a
187 * non-VHE guest hypervisor.
188 */
189 __vcpu_sys_reg(vcpu, reg) = val;
190
191 /* No EL1 counterpart? We're done here.? */
192 if (reg == el1r)
193 return;
194
195 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
196 val = xlate(val);
197
198 /* Redirect this to the EL1 version of the register. */
199 WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r));
200 return;
201 }
202
203 /* EL1 register can't be on the CPU if the guest is in vEL2. */
204 if (unlikely(is_hyp_ctxt(vcpu)))
205 goto memory_write;
206
207 if (__vcpu_write_sys_reg_to_cpu(val, reg))
208 return;
209
210 memory_write:
211 __vcpu_sys_reg(vcpu, reg) = val;
212 }
213
214 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
215 #define CSSELR_MAX 14
216
217 /*
218 * Returns the minimum line size for the selected cache, expressed as
219 * Log2(bytes).
220 */
get_min_cache_line_size(bool icache)221 static u8 get_min_cache_line_size(bool icache)
222 {
223 u64 ctr = read_sanitised_ftr_reg(SYS_CTR_EL0);
224 u8 field;
225
226 if (icache)
227 field = SYS_FIELD_GET(CTR_EL0, IminLine, ctr);
228 else
229 field = SYS_FIELD_GET(CTR_EL0, DminLine, ctr);
230
231 /*
232 * Cache line size is represented as Log2(words) in CTR_EL0.
233 * Log2(bytes) can be derived with the following:
234 *
235 * Log2(words) + 2 = Log2(bytes / 4) + 2
236 * = Log2(bytes) - 2 + 2
237 * = Log2(bytes)
238 */
239 return field + 2;
240 }
241
242 /* Which cache CCSIDR represents depends on CSSELR value. */
get_ccsidr(struct kvm_vcpu * vcpu,u32 csselr)243 static u32 get_ccsidr(struct kvm_vcpu *vcpu, u32 csselr)
244 {
245 u8 line_size;
246
247 if (vcpu->arch.ccsidr)
248 return vcpu->arch.ccsidr[csselr];
249
250 line_size = get_min_cache_line_size(csselr & CSSELR_EL1_InD);
251
252 /*
253 * Fabricate a CCSIDR value as the overriding value does not exist.
254 * The real CCSIDR value will not be used as it can vary by the
255 * physical CPU which the vcpu currently resides in.
256 *
257 * The line size is determined with get_min_cache_line_size(), which
258 * should be valid for all CPUs even if they have different cache
259 * configuration.
260 *
261 * The associativity bits are cleared, meaning the geometry of all data
262 * and unified caches (which are guaranteed to be PIPT and thus
263 * non-aliasing) are 1 set and 1 way.
264 * Guests should not be doing cache operations by set/way at all, and
265 * for this reason, we trap them and attempt to infer the intent, so
266 * that we can flush the entire guest's address space at the appropriate
267 * time. The exposed geometry minimizes the number of the traps.
268 * [If guests should attempt to infer aliasing properties from the
269 * geometry (which is not permitted by the architecture), they would
270 * only do so for virtually indexed caches.]
271 *
272 * We don't check if the cache level exists as it is allowed to return
273 * an UNKNOWN value if not.
274 */
275 return SYS_FIELD_PREP(CCSIDR_EL1, LineSize, line_size - 4);
276 }
277
set_ccsidr(struct kvm_vcpu * vcpu,u32 csselr,u32 val)278 static int set_ccsidr(struct kvm_vcpu *vcpu, u32 csselr, u32 val)
279 {
280 u8 line_size = FIELD_GET(CCSIDR_EL1_LineSize, val) + 4;
281 u32 *ccsidr = vcpu->arch.ccsidr;
282 u32 i;
283
284 if ((val & CCSIDR_EL1_RES0) ||
285 line_size < get_min_cache_line_size(csselr & CSSELR_EL1_InD))
286 return -EINVAL;
287
288 if (!ccsidr) {
289 if (val == get_ccsidr(vcpu, csselr))
290 return 0;
291
292 ccsidr = kmalloc_array(CSSELR_MAX, sizeof(u32), GFP_KERNEL_ACCOUNT);
293 if (!ccsidr)
294 return -ENOMEM;
295
296 for (i = 0; i < CSSELR_MAX; i++)
297 ccsidr[i] = get_ccsidr(vcpu, i);
298
299 vcpu->arch.ccsidr = ccsidr;
300 }
301
302 ccsidr[csselr] = val;
303
304 return 0;
305 }
306
access_rw(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)307 static bool access_rw(struct kvm_vcpu *vcpu,
308 struct sys_reg_params *p,
309 const struct sys_reg_desc *r)
310 {
311 if (p->is_write)
312 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
313 else
314 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
315
316 return true;
317 }
318
319 /*
320 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
321 */
access_dcsw(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)322 static bool access_dcsw(struct kvm_vcpu *vcpu,
323 struct sys_reg_params *p,
324 const struct sys_reg_desc *r)
325 {
326 if (!p->is_write)
327 return read_from_write_only(vcpu, p, r);
328
329 /*
330 * Only track S/W ops if we don't have FWB. It still indicates
331 * that the guest is a bit broken (S/W operations should only
332 * be done by firmware, knowing that there is only a single
333 * CPU left in the system, and certainly not from non-secure
334 * software).
335 */
336 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
337 kvm_set_way_flush(vcpu);
338
339 return true;
340 }
341
access_dcgsw(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)342 static bool access_dcgsw(struct kvm_vcpu *vcpu,
343 struct sys_reg_params *p,
344 const struct sys_reg_desc *r)
345 {
346 if (!kvm_has_mte(vcpu->kvm)) {
347 kvm_inject_undefined(vcpu);
348 return false;
349 }
350
351 /* Treat MTE S/W ops as we treat the classic ones: with contempt */
352 return access_dcsw(vcpu, p, r);
353 }
354
get_access_mask(const struct sys_reg_desc * r,u64 * mask,u64 * shift)355 static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
356 {
357 switch (r->aarch32_map) {
358 case AA32_LO:
359 *mask = GENMASK_ULL(31, 0);
360 *shift = 0;
361 break;
362 case AA32_HI:
363 *mask = GENMASK_ULL(63, 32);
364 *shift = 32;
365 break;
366 default:
367 *mask = GENMASK_ULL(63, 0);
368 *shift = 0;
369 break;
370 }
371 }
372
373 /*
374 * Generic accessor for VM registers. Only called as long as HCR_TVM
375 * is set. If the guest enables the MMU, we stop trapping the VM
376 * sys_regs and leave it in complete control of the caches.
377 */
access_vm_reg(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)378 static bool access_vm_reg(struct kvm_vcpu *vcpu,
379 struct sys_reg_params *p,
380 const struct sys_reg_desc *r)
381 {
382 bool was_enabled = vcpu_has_cache_enabled(vcpu);
383 u64 val, mask, shift;
384
385 BUG_ON(!p->is_write);
386
387 get_access_mask(r, &mask, &shift);
388
389 if (~mask) {
390 val = vcpu_read_sys_reg(vcpu, r->reg);
391 val &= ~mask;
392 } else {
393 val = 0;
394 }
395
396 val |= (p->regval & (mask >> shift)) << shift;
397 vcpu_write_sys_reg(vcpu, val, r->reg);
398
399 kvm_toggle_cache(vcpu, was_enabled);
400 return true;
401 }
402
access_actlr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)403 static bool access_actlr(struct kvm_vcpu *vcpu,
404 struct sys_reg_params *p,
405 const struct sys_reg_desc *r)
406 {
407 u64 mask, shift;
408
409 if (p->is_write)
410 return ignore_write(vcpu, p);
411
412 get_access_mask(r, &mask, &shift);
413 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
414
415 return true;
416 }
417
418 /*
419 * Trap handler for the GICv3 SGI generation system register.
420 * Forward the request to the VGIC emulation.
421 * The cp15_64 code makes sure this automatically works
422 * for both AArch64 and AArch32 accesses.
423 */
access_gic_sgi(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)424 static bool access_gic_sgi(struct kvm_vcpu *vcpu,
425 struct sys_reg_params *p,
426 const struct sys_reg_desc *r)
427 {
428 bool g1;
429
430 if (!p->is_write)
431 return read_from_write_only(vcpu, p, r);
432
433 /*
434 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
435 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
436 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
437 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
438 * group.
439 */
440 if (p->Op0 == 0) { /* AArch32 */
441 switch (p->Op1) {
442 default: /* Keep GCC quiet */
443 case 0: /* ICC_SGI1R */
444 g1 = true;
445 break;
446 case 1: /* ICC_ASGI1R */
447 case 2: /* ICC_SGI0R */
448 g1 = false;
449 break;
450 }
451 } else { /* AArch64 */
452 switch (p->Op2) {
453 default: /* Keep GCC quiet */
454 case 5: /* ICC_SGI1R_EL1 */
455 g1 = true;
456 break;
457 case 6: /* ICC_ASGI1R_EL1 */
458 case 7: /* ICC_SGI0R_EL1 */
459 g1 = false;
460 break;
461 }
462 }
463
464 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
465
466 return true;
467 }
468
access_gic_sre(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)469 static bool access_gic_sre(struct kvm_vcpu *vcpu,
470 struct sys_reg_params *p,
471 const struct sys_reg_desc *r)
472 {
473 if (p->is_write)
474 return ignore_write(vcpu, p);
475
476 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
477 return true;
478 }
479
trap_raz_wi(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)480 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
481 struct sys_reg_params *p,
482 const struct sys_reg_desc *r)
483 {
484 if (p->is_write)
485 return ignore_write(vcpu, p);
486 else
487 return read_zero(vcpu, p);
488 }
489
trap_undef(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)490 static bool trap_undef(struct kvm_vcpu *vcpu,
491 struct sys_reg_params *p,
492 const struct sys_reg_desc *r)
493 {
494 kvm_inject_undefined(vcpu);
495 return false;
496 }
497
498 /*
499 * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
500 * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
501 * system, these registers should UNDEF. LORID_EL1 being a RO register, we
502 * treat it separately.
503 */
trap_loregion(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)504 static bool trap_loregion(struct kvm_vcpu *vcpu,
505 struct sys_reg_params *p,
506 const struct sys_reg_desc *r)
507 {
508 u64 val = IDREG(vcpu->kvm, SYS_ID_AA64MMFR1_EL1);
509 u32 sr = reg_to_encoding(r);
510
511 if (!(val & (0xfUL << ID_AA64MMFR1_EL1_LO_SHIFT))) {
512 kvm_inject_undefined(vcpu);
513 return false;
514 }
515
516 if (p->is_write && sr == SYS_LORID_EL1)
517 return write_to_read_only(vcpu, p, r);
518
519 return trap_raz_wi(vcpu, p, r);
520 }
521
trap_oslar_el1(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)522 static bool trap_oslar_el1(struct kvm_vcpu *vcpu,
523 struct sys_reg_params *p,
524 const struct sys_reg_desc *r)
525 {
526 u64 oslsr;
527
528 if (!p->is_write)
529 return read_from_write_only(vcpu, p, r);
530
531 /* Forward the OSLK bit to OSLSR */
532 oslsr = __vcpu_sys_reg(vcpu, OSLSR_EL1) & ~OSLSR_EL1_OSLK;
533 if (p->regval & OSLAR_EL1_OSLK)
534 oslsr |= OSLSR_EL1_OSLK;
535
536 __vcpu_sys_reg(vcpu, OSLSR_EL1) = oslsr;
537 return true;
538 }
539
trap_oslsr_el1(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)540 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
541 struct sys_reg_params *p,
542 const struct sys_reg_desc *r)
543 {
544 if (p->is_write)
545 return write_to_read_only(vcpu, p, r);
546
547 p->regval = __vcpu_sys_reg(vcpu, r->reg);
548 return true;
549 }
550
set_oslsr_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)551 static int set_oslsr_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
552 u64 val)
553 {
554 /*
555 * The only modifiable bit is the OSLK bit. Refuse the write if
556 * userspace attempts to change any other bit in the register.
557 */
558 if ((val ^ rd->val) & ~OSLSR_EL1_OSLK)
559 return -EINVAL;
560
561 __vcpu_sys_reg(vcpu, rd->reg) = val;
562 return 0;
563 }
564
trap_dbgauthstatus_el1(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)565 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
566 struct sys_reg_params *p,
567 const struct sys_reg_desc *r)
568 {
569 if (p->is_write) {
570 return ignore_write(vcpu, p);
571 } else {
572 p->regval = read_sysreg(dbgauthstatus_el1);
573 return true;
574 }
575 }
576
577 /*
578 * We want to avoid world-switching all the DBG registers all the
579 * time:
580 *
581 * - If we've touched any debug register, it is likely that we're
582 * going to touch more of them. It then makes sense to disable the
583 * traps and start doing the save/restore dance
584 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
585 * then mandatory to save/restore the registers, as the guest
586 * depends on them.
587 *
588 * For this, we use a DIRTY bit, indicating the guest has modified the
589 * debug registers, used as follow:
590 *
591 * On guest entry:
592 * - If the dirty bit is set (because we're coming back from trapping),
593 * disable the traps, save host registers, restore guest registers.
594 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
595 * set the dirty bit, disable the traps, save host registers,
596 * restore guest registers.
597 * - Otherwise, enable the traps
598 *
599 * On guest exit:
600 * - If the dirty bit is set, save guest registers, restore host
601 * registers and clear the dirty bit. This ensure that the host can
602 * now use the debug registers.
603 */
trap_debug_regs(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)604 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
605 struct sys_reg_params *p,
606 const struct sys_reg_desc *r)
607 {
608 access_rw(vcpu, p, r);
609 if (p->is_write)
610 vcpu_set_flag(vcpu, DEBUG_DIRTY);
611
612 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
613
614 return true;
615 }
616
617 /*
618 * reg_to_dbg/dbg_to_reg
619 *
620 * A 32 bit write to a debug register leave top bits alone
621 * A 32 bit read from a debug register only returns the bottom bits
622 *
623 * All writes will set the DEBUG_DIRTY flag to ensure the hyp code
624 * switches between host and guest values in future.
625 */
reg_to_dbg(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * rd,u64 * dbg_reg)626 static void reg_to_dbg(struct kvm_vcpu *vcpu,
627 struct sys_reg_params *p,
628 const struct sys_reg_desc *rd,
629 u64 *dbg_reg)
630 {
631 u64 mask, shift, val;
632
633 get_access_mask(rd, &mask, &shift);
634
635 val = *dbg_reg;
636 val &= ~mask;
637 val |= (p->regval & (mask >> shift)) << shift;
638 *dbg_reg = val;
639
640 vcpu_set_flag(vcpu, DEBUG_DIRTY);
641 }
642
dbg_to_reg(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * rd,u64 * dbg_reg)643 static void dbg_to_reg(struct kvm_vcpu *vcpu,
644 struct sys_reg_params *p,
645 const struct sys_reg_desc *rd,
646 u64 *dbg_reg)
647 {
648 u64 mask, shift;
649
650 get_access_mask(rd, &mask, &shift);
651 p->regval = (*dbg_reg & mask) >> shift;
652 }
653
trap_bvr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * rd)654 static bool trap_bvr(struct kvm_vcpu *vcpu,
655 struct sys_reg_params *p,
656 const struct sys_reg_desc *rd)
657 {
658 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
659
660 if (p->is_write)
661 reg_to_dbg(vcpu, p, rd, dbg_reg);
662 else
663 dbg_to_reg(vcpu, p, rd, dbg_reg);
664
665 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
666
667 return true;
668 }
669
set_bvr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)670 static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
671 u64 val)
672 {
673 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = val;
674 return 0;
675 }
676
get_bvr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 * val)677 static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
678 u64 *val)
679 {
680 *val = vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
681 return 0;
682 }
683
reset_bvr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)684 static u64 reset_bvr(struct kvm_vcpu *vcpu,
685 const struct sys_reg_desc *rd)
686 {
687 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val;
688 return rd->val;
689 }
690
trap_bcr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * rd)691 static bool trap_bcr(struct kvm_vcpu *vcpu,
692 struct sys_reg_params *p,
693 const struct sys_reg_desc *rd)
694 {
695 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
696
697 if (p->is_write)
698 reg_to_dbg(vcpu, p, rd, dbg_reg);
699 else
700 dbg_to_reg(vcpu, p, rd, dbg_reg);
701
702 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
703
704 return true;
705 }
706
set_bcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)707 static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
708 u64 val)
709 {
710 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = val;
711 return 0;
712 }
713
get_bcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 * val)714 static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
715 u64 *val)
716 {
717 *val = vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
718 return 0;
719 }
720
reset_bcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)721 static u64 reset_bcr(struct kvm_vcpu *vcpu,
722 const struct sys_reg_desc *rd)
723 {
724 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val;
725 return rd->val;
726 }
727
trap_wvr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * rd)728 static bool trap_wvr(struct kvm_vcpu *vcpu,
729 struct sys_reg_params *p,
730 const struct sys_reg_desc *rd)
731 {
732 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
733
734 if (p->is_write)
735 reg_to_dbg(vcpu, p, rd, dbg_reg);
736 else
737 dbg_to_reg(vcpu, p, rd, dbg_reg);
738
739 trace_trap_reg(__func__, rd->CRm, p->is_write,
740 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
741
742 return true;
743 }
744
set_wvr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)745 static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
746 u64 val)
747 {
748 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = val;
749 return 0;
750 }
751
get_wvr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 * val)752 static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
753 u64 *val)
754 {
755 *val = vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
756 return 0;
757 }
758
reset_wvr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)759 static u64 reset_wvr(struct kvm_vcpu *vcpu,
760 const struct sys_reg_desc *rd)
761 {
762 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val;
763 return rd->val;
764 }
765
trap_wcr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * rd)766 static bool trap_wcr(struct kvm_vcpu *vcpu,
767 struct sys_reg_params *p,
768 const struct sys_reg_desc *rd)
769 {
770 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
771
772 if (p->is_write)
773 reg_to_dbg(vcpu, p, rd, dbg_reg);
774 else
775 dbg_to_reg(vcpu, p, rd, dbg_reg);
776
777 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
778
779 return true;
780 }
781
set_wcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)782 static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
783 u64 val)
784 {
785 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = val;
786 return 0;
787 }
788
get_wcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 * val)789 static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
790 u64 *val)
791 {
792 *val = vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
793 return 0;
794 }
795
reset_wcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)796 static u64 reset_wcr(struct kvm_vcpu *vcpu,
797 const struct sys_reg_desc *rd)
798 {
799 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val;
800 return rd->val;
801 }
802
reset_amair_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)803 static u64 reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
804 {
805 u64 amair = read_sysreg(amair_el1);
806 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
807 return amair;
808 }
809
reset_actlr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)810 static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
811 {
812 u64 actlr = read_sysreg(actlr_el1);
813 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
814 return actlr;
815 }
816
reset_mpidr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)817 static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
818 {
819 u64 mpidr;
820
821 /*
822 * Map the vcpu_id into the first three affinity level fields of
823 * the MPIDR. We limit the number of VCPUs in level 0 due to a
824 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
825 * of the GICv3 to be able to address each CPU directly when
826 * sending IPIs.
827 */
828 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
829 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
830 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
831 mpidr |= (1ULL << 31);
832 vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1);
833
834 return mpidr;
835 }
836
pmu_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)837 static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
838 const struct sys_reg_desc *r)
839 {
840 if (kvm_vcpu_has_pmu(vcpu))
841 return 0;
842
843 return REG_HIDDEN;
844 }
845
reset_pmu_reg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)846 static u64 reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
847 {
848 u64 mask = BIT(ARMV8_PMU_CYCLE_IDX);
849 u8 n = vcpu->kvm->arch.pmcr_n;
850
851 if (n)
852 mask |= GENMASK(n - 1, 0);
853
854 reset_unknown(vcpu, r);
855 __vcpu_sys_reg(vcpu, r->reg) &= mask;
856
857 return __vcpu_sys_reg(vcpu, r->reg);
858 }
859
reset_pmevcntr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)860 static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
861 {
862 reset_unknown(vcpu, r);
863 __vcpu_sys_reg(vcpu, r->reg) &= GENMASK(31, 0);
864
865 return __vcpu_sys_reg(vcpu, r->reg);
866 }
867
reset_pmevtyper(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)868 static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
869 {
870 /* This thing will UNDEF, who cares about the reset value? */
871 if (!kvm_vcpu_has_pmu(vcpu))
872 return 0;
873
874 reset_unknown(vcpu, r);
875 __vcpu_sys_reg(vcpu, r->reg) &= kvm_pmu_evtyper_mask(vcpu->kvm);
876
877 return __vcpu_sys_reg(vcpu, r->reg);
878 }
879
reset_pmselr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)880 static u64 reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
881 {
882 reset_unknown(vcpu, r);
883 __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_COUNTER_MASK;
884
885 return __vcpu_sys_reg(vcpu, r->reg);
886 }
887
reset_pmcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)888 static u64 reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
889 {
890 u64 pmcr = 0;
891
892 if (!kvm_supports_32bit_el0())
893 pmcr |= ARMV8_PMU_PMCR_LC;
894
895 /*
896 * The value of PMCR.N field is included when the
897 * vCPU register is read via kvm_vcpu_read_pmcr().
898 */
899 __vcpu_sys_reg(vcpu, r->reg) = pmcr;
900
901 return __vcpu_sys_reg(vcpu, r->reg);
902 }
903
check_pmu_access_disabled(struct kvm_vcpu * vcpu,u64 flags)904 static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
905 {
906 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
907 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
908
909 if (!enabled)
910 kvm_inject_undefined(vcpu);
911
912 return !enabled;
913 }
914
pmu_access_el0_disabled(struct kvm_vcpu * vcpu)915 static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
916 {
917 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
918 }
919
pmu_write_swinc_el0_disabled(struct kvm_vcpu * vcpu)920 static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
921 {
922 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
923 }
924
pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu * vcpu)925 static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
926 {
927 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
928 }
929
pmu_access_event_counter_el0_disabled(struct kvm_vcpu * vcpu)930 static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
931 {
932 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
933 }
934
access_pmcr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)935 static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
936 const struct sys_reg_desc *r)
937 {
938 u64 val;
939
940 if (pmu_access_el0_disabled(vcpu))
941 return false;
942
943 if (p->is_write) {
944 /*
945 * Only update writeable bits of PMCR (continuing into
946 * kvm_pmu_handle_pmcr() as well)
947 */
948 val = kvm_vcpu_read_pmcr(vcpu);
949 val &= ~ARMV8_PMU_PMCR_MASK;
950 val |= p->regval & ARMV8_PMU_PMCR_MASK;
951 if (!kvm_supports_32bit_el0())
952 val |= ARMV8_PMU_PMCR_LC;
953 kvm_pmu_handle_pmcr(vcpu, val);
954 } else {
955 /* PMCR.P & PMCR.C are RAZ */
956 val = kvm_vcpu_read_pmcr(vcpu)
957 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
958 p->regval = val;
959 }
960
961 return true;
962 }
963
access_pmselr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)964 static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
965 const struct sys_reg_desc *r)
966 {
967 if (pmu_access_event_counter_el0_disabled(vcpu))
968 return false;
969
970 if (p->is_write)
971 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
972 else
973 /* return PMSELR.SEL field */
974 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
975 & ARMV8_PMU_COUNTER_MASK;
976
977 return true;
978 }
979
access_pmceid(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)980 static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
981 const struct sys_reg_desc *r)
982 {
983 u64 pmceid, mask, shift;
984
985 BUG_ON(p->is_write);
986
987 if (pmu_access_el0_disabled(vcpu))
988 return false;
989
990 get_access_mask(r, &mask, &shift);
991
992 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
993 pmceid &= mask;
994 pmceid >>= shift;
995
996 p->regval = pmceid;
997
998 return true;
999 }
1000
pmu_counter_idx_valid(struct kvm_vcpu * vcpu,u64 idx)1001 static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
1002 {
1003 u64 pmcr, val;
1004
1005 pmcr = kvm_vcpu_read_pmcr(vcpu);
1006 val = FIELD_GET(ARMV8_PMU_PMCR_N, pmcr);
1007 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
1008 kvm_inject_undefined(vcpu);
1009 return false;
1010 }
1011
1012 return true;
1013 }
1014
get_pmu_evcntr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r,u64 * val)1015 static int get_pmu_evcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1016 u64 *val)
1017 {
1018 u64 idx;
1019
1020 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 0)
1021 /* PMCCNTR_EL0 */
1022 idx = ARMV8_PMU_CYCLE_IDX;
1023 else
1024 /* PMEVCNTRn_EL0 */
1025 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1026
1027 *val = kvm_pmu_get_counter_value(vcpu, idx);
1028 return 0;
1029 }
1030
access_pmu_evcntr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1031 static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
1032 struct sys_reg_params *p,
1033 const struct sys_reg_desc *r)
1034 {
1035 u64 idx = ~0UL;
1036
1037 if (r->CRn == 9 && r->CRm == 13) {
1038 if (r->Op2 == 2) {
1039 /* PMXEVCNTR_EL0 */
1040 if (pmu_access_event_counter_el0_disabled(vcpu))
1041 return false;
1042
1043 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
1044 & ARMV8_PMU_COUNTER_MASK;
1045 } else if (r->Op2 == 0) {
1046 /* PMCCNTR_EL0 */
1047 if (pmu_access_cycle_counter_el0_disabled(vcpu))
1048 return false;
1049
1050 idx = ARMV8_PMU_CYCLE_IDX;
1051 }
1052 } else if (r->CRn == 0 && r->CRm == 9) {
1053 /* PMCCNTR */
1054 if (pmu_access_event_counter_el0_disabled(vcpu))
1055 return false;
1056
1057 idx = ARMV8_PMU_CYCLE_IDX;
1058 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
1059 /* PMEVCNTRn_EL0 */
1060 if (pmu_access_event_counter_el0_disabled(vcpu))
1061 return false;
1062
1063 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1064 }
1065
1066 /* Catch any decoding mistake */
1067 WARN_ON(idx == ~0UL);
1068
1069 if (!pmu_counter_idx_valid(vcpu, idx))
1070 return false;
1071
1072 if (p->is_write) {
1073 if (pmu_access_el0_disabled(vcpu))
1074 return false;
1075
1076 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
1077 } else {
1078 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
1079 }
1080
1081 return true;
1082 }
1083
access_pmu_evtyper(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1084 static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1085 const struct sys_reg_desc *r)
1086 {
1087 u64 idx, reg;
1088
1089 if (pmu_access_el0_disabled(vcpu))
1090 return false;
1091
1092 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
1093 /* PMXEVTYPER_EL0 */
1094 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
1095 reg = PMEVTYPER0_EL0 + idx;
1096 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
1097 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1098 if (idx == ARMV8_PMU_CYCLE_IDX)
1099 reg = PMCCFILTR_EL0;
1100 else
1101 /* PMEVTYPERn_EL0 */
1102 reg = PMEVTYPER0_EL0 + idx;
1103 } else {
1104 BUG();
1105 }
1106
1107 if (!pmu_counter_idx_valid(vcpu, idx))
1108 return false;
1109
1110 if (p->is_write) {
1111 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
1112 kvm_vcpu_pmu_restore_guest(vcpu);
1113 } else {
1114 p->regval = __vcpu_sys_reg(vcpu, reg);
1115 }
1116
1117 return true;
1118 }
1119
set_pmreg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r,u64 val)1120 static int set_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 val)
1121 {
1122 bool set;
1123
1124 val &= kvm_pmu_valid_counter_mask(vcpu);
1125
1126 switch (r->reg) {
1127 case PMOVSSET_EL0:
1128 /* CRm[1] being set indicates a SET register, and CLR otherwise */
1129 set = r->CRm & 2;
1130 break;
1131 default:
1132 /* Op2[0] being set indicates a SET register, and CLR otherwise */
1133 set = r->Op2 & 1;
1134 break;
1135 }
1136
1137 if (set)
1138 __vcpu_sys_reg(vcpu, r->reg) |= val;
1139 else
1140 __vcpu_sys_reg(vcpu, r->reg) &= ~val;
1141
1142 return 0;
1143 }
1144
get_pmreg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r,u64 * val)1145 static int get_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 *val)
1146 {
1147 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1148
1149 *val = __vcpu_sys_reg(vcpu, r->reg) & mask;
1150 return 0;
1151 }
1152
access_pmcnten(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1153 static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1154 const struct sys_reg_desc *r)
1155 {
1156 u64 val, mask;
1157
1158 if (pmu_access_el0_disabled(vcpu))
1159 return false;
1160
1161 mask = kvm_pmu_valid_counter_mask(vcpu);
1162 if (p->is_write) {
1163 val = p->regval & mask;
1164 if (r->Op2 & 0x1) {
1165 /* accessing PMCNTENSET_EL0 */
1166 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
1167 kvm_pmu_enable_counter_mask(vcpu, val);
1168 kvm_vcpu_pmu_restore_guest(vcpu);
1169 } else {
1170 /* accessing PMCNTENCLR_EL0 */
1171 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
1172 kvm_pmu_disable_counter_mask(vcpu, val);
1173 }
1174 } else {
1175 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
1176 }
1177
1178 return true;
1179 }
1180
access_pminten(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1181 static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1182 const struct sys_reg_desc *r)
1183 {
1184 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1185
1186 if (check_pmu_access_disabled(vcpu, 0))
1187 return false;
1188
1189 if (p->is_write) {
1190 u64 val = p->regval & mask;
1191
1192 if (r->Op2 & 0x1)
1193 /* accessing PMINTENSET_EL1 */
1194 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
1195 else
1196 /* accessing PMINTENCLR_EL1 */
1197 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
1198 } else {
1199 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
1200 }
1201
1202 return true;
1203 }
1204
access_pmovs(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1205 static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1206 const struct sys_reg_desc *r)
1207 {
1208 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1209
1210 if (pmu_access_el0_disabled(vcpu))
1211 return false;
1212
1213 if (p->is_write) {
1214 if (r->CRm & 0x2)
1215 /* accessing PMOVSSET_EL0 */
1216 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
1217 else
1218 /* accessing PMOVSCLR_EL0 */
1219 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
1220 } else {
1221 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
1222 }
1223
1224 return true;
1225 }
1226
access_pmswinc(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1227 static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1228 const struct sys_reg_desc *r)
1229 {
1230 u64 mask;
1231
1232 if (!p->is_write)
1233 return read_from_write_only(vcpu, p, r);
1234
1235 if (pmu_write_swinc_el0_disabled(vcpu))
1236 return false;
1237
1238 mask = kvm_pmu_valid_counter_mask(vcpu);
1239 kvm_pmu_software_increment(vcpu, p->regval & mask);
1240 return true;
1241 }
1242
access_pmuserenr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1243 static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1244 const struct sys_reg_desc *r)
1245 {
1246 if (p->is_write) {
1247 if (!vcpu_mode_priv(vcpu)) {
1248 kvm_inject_undefined(vcpu);
1249 return false;
1250 }
1251
1252 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
1253 p->regval & ARMV8_PMU_USERENR_MASK;
1254 } else {
1255 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
1256 & ARMV8_PMU_USERENR_MASK;
1257 }
1258
1259 return true;
1260 }
1261
get_pmcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r,u64 * val)1262 static int get_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1263 u64 *val)
1264 {
1265 *val = kvm_vcpu_read_pmcr(vcpu);
1266 return 0;
1267 }
1268
set_pmcr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r,u64 val)1269 static int set_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1270 u64 val)
1271 {
1272 u8 new_n = FIELD_GET(ARMV8_PMU_PMCR_N, val);
1273 struct kvm *kvm = vcpu->kvm;
1274
1275 mutex_lock(&kvm->arch.config_lock);
1276
1277 /*
1278 * The vCPU can't have more counters than the PMU hardware
1279 * implements. Ignore this error to maintain compatibility
1280 * with the existing KVM behavior.
1281 */
1282 if (!kvm_vm_has_ran_once(kvm) &&
1283 new_n <= kvm_arm_pmu_get_max_counters(kvm))
1284 kvm->arch.pmcr_n = new_n;
1285
1286 mutex_unlock(&kvm->arch.config_lock);
1287
1288 /*
1289 * Ignore writes to RES0 bits, read only bits that are cleared on
1290 * vCPU reset, and writable bits that KVM doesn't support yet.
1291 * (i.e. only PMCR.N and bits [7:0] are mutable from userspace)
1292 * The LP bit is RES0 when FEAT_PMUv3p5 is not supported on the vCPU.
1293 * But, we leave the bit as it is here, as the vCPU's PMUver might
1294 * be changed later (NOTE: the bit will be cleared on first vCPU run
1295 * if necessary).
1296 */
1297 val &= ARMV8_PMU_PMCR_MASK;
1298
1299 /* The LC bit is RES1 when AArch32 is not supported */
1300 if (!kvm_supports_32bit_el0())
1301 val |= ARMV8_PMU_PMCR_LC;
1302
1303 __vcpu_sys_reg(vcpu, r->reg) = val;
1304 return 0;
1305 }
1306
1307 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
1308 #define DBG_BCR_BVR_WCR_WVR_EL1(n) \
1309 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
1310 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
1311 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
1312 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
1313 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
1314 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
1315 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
1316 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
1317
1318 #define PMU_SYS_REG(name) \
1319 SYS_DESC(SYS_##name), .reset = reset_pmu_reg, \
1320 .visibility = pmu_visibility
1321
1322 /* Macro to expand the PMEVCNTRn_EL0 register */
1323 #define PMU_PMEVCNTR_EL0(n) \
1324 { PMU_SYS_REG(PMEVCNTRn_EL0(n)), \
1325 .reset = reset_pmevcntr, .get_user = get_pmu_evcntr, \
1326 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
1327
1328 /* Macro to expand the PMEVTYPERn_EL0 register */
1329 #define PMU_PMEVTYPER_EL0(n) \
1330 { PMU_SYS_REG(PMEVTYPERn_EL0(n)), \
1331 .reset = reset_pmevtyper, \
1332 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
1333
undef_access(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1334 static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1335 const struct sys_reg_desc *r)
1336 {
1337 kvm_inject_undefined(vcpu);
1338
1339 return false;
1340 }
1341
1342 /* Macro to expand the AMU counter and type registers*/
1343 #define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
1344 #define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
1345 #define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
1346 #define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
1347
ptrauth_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)1348 static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
1349 const struct sys_reg_desc *rd)
1350 {
1351 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
1352 }
1353
1354 /*
1355 * If we land here on a PtrAuth access, that is because we didn't
1356 * fixup the access on exit by allowing the PtrAuth sysregs. The only
1357 * way this happens is when the guest does not have PtrAuth support
1358 * enabled.
1359 */
1360 #define __PTRAUTH_KEY(k) \
1361 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
1362 .visibility = ptrauth_visibility}
1363
1364 #define PTRAUTH_KEY(k) \
1365 __PTRAUTH_KEY(k ## KEYLO_EL1), \
1366 __PTRAUTH_KEY(k ## KEYHI_EL1)
1367
access_arch_timer(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1368 static bool access_arch_timer(struct kvm_vcpu *vcpu,
1369 struct sys_reg_params *p,
1370 const struct sys_reg_desc *r)
1371 {
1372 enum kvm_arch_timers tmr;
1373 enum kvm_arch_timer_regs treg;
1374 u64 reg = reg_to_encoding(r);
1375
1376 switch (reg) {
1377 case SYS_CNTP_TVAL_EL0:
1378 case SYS_AARCH32_CNTP_TVAL:
1379 tmr = TIMER_PTIMER;
1380 treg = TIMER_REG_TVAL;
1381 break;
1382 case SYS_CNTP_CTL_EL0:
1383 case SYS_AARCH32_CNTP_CTL:
1384 tmr = TIMER_PTIMER;
1385 treg = TIMER_REG_CTL;
1386 break;
1387 case SYS_CNTP_CVAL_EL0:
1388 case SYS_AARCH32_CNTP_CVAL:
1389 tmr = TIMER_PTIMER;
1390 treg = TIMER_REG_CVAL;
1391 break;
1392 case SYS_CNTPCT_EL0:
1393 case SYS_CNTPCTSS_EL0:
1394 case SYS_AARCH32_CNTPCT:
1395 tmr = TIMER_PTIMER;
1396 treg = TIMER_REG_CNT;
1397 break;
1398 default:
1399 print_sys_reg_msg(p, "%s", "Unhandled trapped timer register");
1400 kvm_inject_undefined(vcpu);
1401 return false;
1402 }
1403
1404 if (p->is_write)
1405 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1406 else
1407 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1408
1409 return true;
1410 }
1411
kvm_arm64_ftr_safe_value(u32 id,const struct arm64_ftr_bits * ftrp,s64 new,s64 cur)1412 static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,
1413 s64 new, s64 cur)
1414 {
1415 struct arm64_ftr_bits kvm_ftr = *ftrp;
1416
1417 /* Some features have different safe value type in KVM than host features */
1418 switch (id) {
1419 case SYS_ID_AA64DFR0_EL1:
1420 switch (kvm_ftr.shift) {
1421 case ID_AA64DFR0_EL1_PMUVer_SHIFT:
1422 kvm_ftr.type = FTR_LOWER_SAFE;
1423 break;
1424 case ID_AA64DFR0_EL1_DebugVer_SHIFT:
1425 kvm_ftr.type = FTR_LOWER_SAFE;
1426 break;
1427 }
1428 break;
1429 case SYS_ID_DFR0_EL1:
1430 if (kvm_ftr.shift == ID_DFR0_EL1_PerfMon_SHIFT)
1431 kvm_ftr.type = FTR_LOWER_SAFE;
1432 break;
1433 }
1434
1435 return arm64_ftr_safe_value(&kvm_ftr, new, cur);
1436 }
1437
1438 /*
1439 * arm64_check_features() - Check if a feature register value constitutes
1440 * a subset of features indicated by the idreg's KVM sanitised limit.
1441 *
1442 * This function will check if each feature field of @val is the "safe" value
1443 * against idreg's KVM sanitised limit return from reset() callback.
1444 * If a field value in @val is the same as the one in limit, it is always
1445 * considered the safe value regardless For register fields that are not in
1446 * writable, only the value in limit is considered the safe value.
1447 *
1448 * Return: 0 if all the fields are safe. Otherwise, return negative errno.
1449 */
arm64_check_features(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)1450 static int arm64_check_features(struct kvm_vcpu *vcpu,
1451 const struct sys_reg_desc *rd,
1452 u64 val)
1453 {
1454 const struct arm64_ftr_reg *ftr_reg;
1455 const struct arm64_ftr_bits *ftrp = NULL;
1456 u32 id = reg_to_encoding(rd);
1457 u64 writable_mask = rd->val;
1458 u64 limit = rd->reset(vcpu, rd);
1459 u64 mask = 0;
1460
1461 /*
1462 * Hidden and unallocated ID registers may not have a corresponding
1463 * struct arm64_ftr_reg. Of course, if the register is RAZ we know the
1464 * only safe value is 0.
1465 */
1466 if (sysreg_visible_as_raz(vcpu, rd))
1467 return val ? -E2BIG : 0;
1468
1469 ftr_reg = get_arm64_ftr_reg(id);
1470 if (!ftr_reg)
1471 return -EINVAL;
1472
1473 ftrp = ftr_reg->ftr_bits;
1474
1475 for (; ftrp && ftrp->width; ftrp++) {
1476 s64 f_val, f_lim, safe_val;
1477 u64 ftr_mask;
1478
1479 ftr_mask = arm64_ftr_mask(ftrp);
1480 if ((ftr_mask & writable_mask) != ftr_mask)
1481 continue;
1482
1483 f_val = arm64_ftr_value(ftrp, val);
1484 f_lim = arm64_ftr_value(ftrp, limit);
1485 mask |= ftr_mask;
1486
1487 if (f_val == f_lim)
1488 safe_val = f_val;
1489 else
1490 safe_val = kvm_arm64_ftr_safe_value(id, ftrp, f_val, f_lim);
1491
1492 if (safe_val != f_val)
1493 return -E2BIG;
1494 }
1495
1496 /* For fields that are not writable, values in limit are the safe values. */
1497 if ((val & ~mask) != (limit & ~mask))
1498 return -E2BIG;
1499
1500 return 0;
1501 }
1502
pmuver_to_perfmon(u8 pmuver)1503 static u8 pmuver_to_perfmon(u8 pmuver)
1504 {
1505 switch (pmuver) {
1506 case ID_AA64DFR0_EL1_PMUVer_IMP:
1507 return ID_DFR0_EL1_PerfMon_PMUv3;
1508 case ID_AA64DFR0_EL1_PMUVer_IMP_DEF:
1509 return ID_DFR0_EL1_PerfMon_IMPDEF;
1510 default:
1511 /* Anything ARMv8.1+ and NI have the same value. For now. */
1512 return pmuver;
1513 }
1514 }
1515
1516 /* Read a sanitised cpufeature ID register by sys_reg_desc */
__kvm_read_sanitised_id_reg(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1517 static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
1518 const struct sys_reg_desc *r)
1519 {
1520 u32 id = reg_to_encoding(r);
1521 u64 val;
1522
1523 if (sysreg_visible_as_raz(vcpu, r))
1524 return 0;
1525
1526 val = read_sanitised_ftr_reg(id);
1527
1528 switch (id) {
1529 case SYS_ID_AA64PFR1_EL1:
1530 if (!kvm_has_mte(vcpu->kvm))
1531 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
1532
1533 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME);
1534 break;
1535 case SYS_ID_AA64ISAR1_EL1:
1536 if (!vcpu_has_ptrauth(vcpu))
1537 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) |
1538 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) |
1539 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) |
1540 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI));
1541 break;
1542 case SYS_ID_AA64ISAR2_EL1:
1543 if (!vcpu_has_ptrauth(vcpu))
1544 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) |
1545 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
1546 if (!cpus_have_final_cap(ARM64_HAS_WFXT))
1547 val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
1548 break;
1549 case SYS_ID_AA64MMFR2_EL1:
1550 val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
1551 break;
1552 case SYS_ID_MMFR4_EL1:
1553 val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
1554 break;
1555 }
1556
1557 return val;
1558 }
1559
kvm_read_sanitised_id_reg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1560 static u64 kvm_read_sanitised_id_reg(struct kvm_vcpu *vcpu,
1561 const struct sys_reg_desc *r)
1562 {
1563 return __kvm_read_sanitised_id_reg(vcpu, r);
1564 }
1565
read_id_reg(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1566 static u64 read_id_reg(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1567 {
1568 return IDREG(vcpu->kvm, reg_to_encoding(r));
1569 }
1570
1571 /*
1572 * Return true if the register's (Op0, Op1, CRn, CRm, Op2) is
1573 * (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8.
1574 */
is_id_reg(u32 id)1575 static inline bool is_id_reg(u32 id)
1576 {
1577 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1578 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1579 sys_reg_CRm(id) < 8);
1580 }
1581
is_aa32_id_reg(u32 id)1582 static inline bool is_aa32_id_reg(u32 id)
1583 {
1584 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1585 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1586 sys_reg_CRm(id) <= 3);
1587 }
1588
id_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1589 static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1590 const struct sys_reg_desc *r)
1591 {
1592 u32 id = reg_to_encoding(r);
1593
1594 switch (id) {
1595 case SYS_ID_AA64ZFR0_EL1:
1596 if (!vcpu_has_sve(vcpu))
1597 return REG_RAZ;
1598 break;
1599 }
1600
1601 return 0;
1602 }
1603
aa32_id_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1604 static unsigned int aa32_id_visibility(const struct kvm_vcpu *vcpu,
1605 const struct sys_reg_desc *r)
1606 {
1607 /*
1608 * AArch32 ID registers are UNKNOWN if AArch32 isn't implemented at any
1609 * EL. Promote to RAZ/WI in order to guarantee consistency between
1610 * systems.
1611 */
1612 if (!kvm_supports_32bit_el0())
1613 return REG_RAZ | REG_USER_WI;
1614
1615 return id_visibility(vcpu, r);
1616 }
1617
raz_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1618 static unsigned int raz_visibility(const struct kvm_vcpu *vcpu,
1619 const struct sys_reg_desc *r)
1620 {
1621 return REG_RAZ;
1622 }
1623
1624 /* cpufeature ID register access trap handlers */
1625
access_id_reg(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1626 static bool access_id_reg(struct kvm_vcpu *vcpu,
1627 struct sys_reg_params *p,
1628 const struct sys_reg_desc *r)
1629 {
1630 if (p->is_write)
1631 return write_to_read_only(vcpu, p, r);
1632
1633 p->regval = read_id_reg(vcpu, r);
1634
1635 return true;
1636 }
1637
1638 /* Visibility overrides for SVE-specific control registers */
sve_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)1639 static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1640 const struct sys_reg_desc *rd)
1641 {
1642 if (vcpu_has_sve(vcpu))
1643 return 0;
1644
1645 return REG_HIDDEN;
1646 }
1647
read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)1648 static u64 read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1649 const struct sys_reg_desc *rd)
1650 {
1651 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1652
1653 if (!vcpu_has_sve(vcpu))
1654 val &= ~ID_AA64PFR0_EL1_SVE_MASK;
1655
1656 /*
1657 * The default is to expose CSV2 == 1 if the HW isn't affected.
1658 * Although this is a per-CPU feature, we make it global because
1659 * asymmetric systems are just a nuisance.
1660 *
1661 * Userspace can override this as long as it doesn't promise
1662 * the impossible.
1663 */
1664 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) {
1665 val &= ~ID_AA64PFR0_EL1_CSV2_MASK;
1666 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV2, IMP);
1667 }
1668 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) {
1669 val &= ~ID_AA64PFR0_EL1_CSV3_MASK;
1670 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
1671 }
1672
1673 if (kvm_vgic_global_state.type == VGIC_V3) {
1674 val &= ~ID_AA64PFR0_EL1_GIC_MASK;
1675 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
1676 }
1677
1678 val &= ~ID_AA64PFR0_EL1_AMU_MASK;
1679
1680 return val;
1681 }
1682
1683 #define ID_REG_LIMIT_FIELD_ENUM(val, reg, field, limit) \
1684 ({ \
1685 u64 __f_val = FIELD_GET(reg##_##field##_MASK, val); \
1686 (val) &= ~reg##_##field##_MASK; \
1687 (val) |= FIELD_PREP(reg##_##field##_MASK, \
1688 min(__f_val, (u64)reg##_##field##_##limit)); \
1689 (val); \
1690 })
1691
read_sanitised_id_aa64dfr0_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)1692 static u64 read_sanitised_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1693 const struct sys_reg_desc *rd)
1694 {
1695 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1696
1697 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
1698
1699 /*
1700 * Only initialize the PMU version if the vCPU was configured with one.
1701 */
1702 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1703 if (kvm_vcpu_has_pmu(vcpu))
1704 val |= SYS_FIELD_PREP(ID_AA64DFR0_EL1, PMUVer,
1705 kvm_arm_pmu_get_pmuver_limit());
1706
1707 /* Hide SPE from guests */
1708 val &= ~ID_AA64DFR0_EL1_PMSVer_MASK;
1709
1710 return val;
1711 }
1712
set_id_aa64dfr0_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)1713 static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1714 const struct sys_reg_desc *rd,
1715 u64 val)
1716 {
1717 u8 debugver = SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, val);
1718 u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, val);
1719
1720 /*
1721 * Prior to commit 3d0dba5764b9 ("KVM: arm64: PMU: Move the
1722 * ID_AA64DFR0_EL1.PMUver limit to VM creation"), KVM erroneously
1723 * exposed an IMP_DEF PMU to userspace and the guest on systems w/
1724 * non-architectural PMUs. Of course, PMUv3 is the only game in town for
1725 * PMU virtualization, so the IMP_DEF value was rather user-hostile.
1726 *
1727 * At minimum, we're on the hook to allow values that were given to
1728 * userspace by KVM. Cover our tracks here and replace the IMP_DEF value
1729 * with a more sensible NI. The value of an ID register changing under
1730 * the nose of the guest is unfortunate, but is certainly no more
1731 * surprising than an ill-guided PMU driver poking at impdef system
1732 * registers that end in an UNDEF...
1733 */
1734 if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
1735 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1736
1737 /*
1738 * ID_AA64DFR0_EL1.DebugVer is one of those awkward fields with a
1739 * nonzero minimum safe value.
1740 */
1741 if (debugver < ID_AA64DFR0_EL1_DebugVer_IMP)
1742 return -EINVAL;
1743
1744 return set_id_reg(vcpu, rd, val);
1745 }
1746
read_sanitised_id_dfr0_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)1747 static u64 read_sanitised_id_dfr0_el1(struct kvm_vcpu *vcpu,
1748 const struct sys_reg_desc *rd)
1749 {
1750 u8 perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit());
1751 u64 val = read_sanitised_ftr_reg(SYS_ID_DFR0_EL1);
1752
1753 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1754 if (kvm_vcpu_has_pmu(vcpu))
1755 val |= SYS_FIELD_PREP(ID_DFR0_EL1, PerfMon, perfmon);
1756
1757 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_DFR0_EL1, CopDbg, Debugv8p8);
1758
1759 return val;
1760 }
1761
set_id_dfr0_el1(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)1762 static int set_id_dfr0_el1(struct kvm_vcpu *vcpu,
1763 const struct sys_reg_desc *rd,
1764 u64 val)
1765 {
1766 u8 perfmon = SYS_FIELD_GET(ID_DFR0_EL1, PerfMon, val);
1767 u8 copdbg = SYS_FIELD_GET(ID_DFR0_EL1, CopDbg, val);
1768
1769 if (perfmon == ID_DFR0_EL1_PerfMon_IMPDEF) {
1770 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1771 perfmon = 0;
1772 }
1773
1774 /*
1775 * Allow DFR0_EL1.PerfMon to be set from userspace as long as
1776 * it doesn't promise more than what the HW gives us on the
1777 * AArch64 side (as everything is emulated with that), and
1778 * that this is a PMUv3.
1779 */
1780 if (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3)
1781 return -EINVAL;
1782
1783 if (copdbg < ID_DFR0_EL1_CopDbg_Armv8)
1784 return -EINVAL;
1785
1786 return set_id_reg(vcpu, rd, val);
1787 }
1788
1789 /*
1790 * cpufeature ID register user accessors
1791 *
1792 * For now, these registers are immutable for userspace, so no values
1793 * are stored, and for set_id_reg() we don't allow the effective value
1794 * to be changed.
1795 */
get_id_reg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 * val)1796 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1797 u64 *val)
1798 {
1799 /*
1800 * Avoid locking if the VM has already started, as the ID registers are
1801 * guaranteed to be invariant at that point.
1802 */
1803 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1804 *val = read_id_reg(vcpu, rd);
1805 return 0;
1806 }
1807
1808 mutex_lock(&vcpu->kvm->arch.config_lock);
1809 *val = read_id_reg(vcpu, rd);
1810 mutex_unlock(&vcpu->kvm->arch.config_lock);
1811
1812 return 0;
1813 }
1814
set_id_reg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)1815 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1816 u64 val)
1817 {
1818 u32 id = reg_to_encoding(rd);
1819 int ret;
1820
1821 mutex_lock(&vcpu->kvm->arch.config_lock);
1822
1823 /*
1824 * Once the VM has started the ID registers are immutable. Reject any
1825 * write that does not match the final register value.
1826 */
1827 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1828 if (val != read_id_reg(vcpu, rd))
1829 ret = -EBUSY;
1830 else
1831 ret = 0;
1832
1833 mutex_unlock(&vcpu->kvm->arch.config_lock);
1834 return ret;
1835 }
1836
1837 ret = arm64_check_features(vcpu, rd, val);
1838 if (!ret)
1839 IDREG(vcpu->kvm, id) = val;
1840
1841 mutex_unlock(&vcpu->kvm->arch.config_lock);
1842
1843 /*
1844 * arm64_check_features() returns -E2BIG to indicate the register's
1845 * feature set is a superset of the maximally-allowed register value.
1846 * While it would be nice to precisely describe this to userspace, the
1847 * existing UAPI for KVM_SET_ONE_REG has it that invalid register
1848 * writes return -EINVAL.
1849 */
1850 if (ret == -E2BIG)
1851 ret = -EINVAL;
1852 return ret;
1853 }
1854
get_raz_reg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 * val)1855 static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1856 u64 *val)
1857 {
1858 *val = 0;
1859 return 0;
1860 }
1861
set_wi_reg(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)1862 static int set_wi_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1863 u64 val)
1864 {
1865 return 0;
1866 }
1867
access_ctr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1868 static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1869 const struct sys_reg_desc *r)
1870 {
1871 if (p->is_write)
1872 return write_to_read_only(vcpu, p, r);
1873
1874 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1875 return true;
1876 }
1877
access_clidr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1878 static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1879 const struct sys_reg_desc *r)
1880 {
1881 if (p->is_write)
1882 return write_to_read_only(vcpu, p, r);
1883
1884 p->regval = __vcpu_sys_reg(vcpu, r->reg);
1885 return true;
1886 }
1887
1888 /*
1889 * Fabricate a CLIDR_EL1 value instead of using the real value, which can vary
1890 * by the physical CPU which the vcpu currently resides in.
1891 */
reset_clidr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * r)1892 static u64 reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1893 {
1894 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1895 u64 clidr;
1896 u8 loc;
1897
1898 if ((ctr_el0 & CTR_EL0_IDC)) {
1899 /*
1900 * Data cache clean to the PoU is not required so LoUU and LoUIS
1901 * will not be set and a unified cache, which will be marked as
1902 * LoC, will be added.
1903 *
1904 * If not DIC, let the unified cache L2 so that an instruction
1905 * cache can be added as L1 later.
1906 */
1907 loc = (ctr_el0 & CTR_EL0_DIC) ? 1 : 2;
1908 clidr = CACHE_TYPE_UNIFIED << CLIDR_CTYPE_SHIFT(loc);
1909 } else {
1910 /*
1911 * Data cache clean to the PoU is required so let L1 have a data
1912 * cache and mark it as LoUU and LoUIS. As L1 has a data cache,
1913 * it can be marked as LoC too.
1914 */
1915 loc = 1;
1916 clidr = 1 << CLIDR_LOUU_SHIFT;
1917 clidr |= 1 << CLIDR_LOUIS_SHIFT;
1918 clidr |= CACHE_TYPE_DATA << CLIDR_CTYPE_SHIFT(1);
1919 }
1920
1921 /*
1922 * Instruction cache invalidation to the PoU is required so let L1 have
1923 * an instruction cache. If L1 already has a data cache, it will be
1924 * CACHE_TYPE_SEPARATE.
1925 */
1926 if (!(ctr_el0 & CTR_EL0_DIC))
1927 clidr |= CACHE_TYPE_INST << CLIDR_CTYPE_SHIFT(1);
1928
1929 clidr |= loc << CLIDR_LOC_SHIFT;
1930
1931 /*
1932 * Add tag cache unified to data cache. Allocation tags and data are
1933 * unified in a cache line so that it looks valid even if there is only
1934 * one cache line.
1935 */
1936 if (kvm_has_mte(vcpu->kvm))
1937 clidr |= 2 << CLIDR_TTYPE_SHIFT(loc);
1938
1939 __vcpu_sys_reg(vcpu, r->reg) = clidr;
1940
1941 return __vcpu_sys_reg(vcpu, r->reg);
1942 }
1943
set_clidr(struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 val)1944 static int set_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1945 u64 val)
1946 {
1947 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1948 u64 idc = !CLIDR_LOC(val) || (!CLIDR_LOUIS(val) && !CLIDR_LOUU(val));
1949
1950 if ((val & CLIDR_EL1_RES0) || (!(ctr_el0 & CTR_EL0_IDC) && idc))
1951 return -EINVAL;
1952
1953 __vcpu_sys_reg(vcpu, rd->reg) = val;
1954
1955 return 0;
1956 }
1957
access_csselr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1958 static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1959 const struct sys_reg_desc *r)
1960 {
1961 int reg = r->reg;
1962
1963 if (p->is_write)
1964 vcpu_write_sys_reg(vcpu, p->regval, reg);
1965 else
1966 p->regval = vcpu_read_sys_reg(vcpu, reg);
1967 return true;
1968 }
1969
access_ccsidr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)1970 static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1971 const struct sys_reg_desc *r)
1972 {
1973 u32 csselr;
1974
1975 if (p->is_write)
1976 return write_to_read_only(vcpu, p, r);
1977
1978 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1979 csselr &= CSSELR_EL1_Level | CSSELR_EL1_InD;
1980 if (csselr < CSSELR_MAX)
1981 p->regval = get_ccsidr(vcpu, csselr);
1982
1983 return true;
1984 }
1985
mte_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)1986 static unsigned int mte_visibility(const struct kvm_vcpu *vcpu,
1987 const struct sys_reg_desc *rd)
1988 {
1989 if (kvm_has_mte(vcpu->kvm))
1990 return 0;
1991
1992 return REG_HIDDEN;
1993 }
1994
1995 #define MTE_REG(name) { \
1996 SYS_DESC(SYS_##name), \
1997 .access = undef_access, \
1998 .reset = reset_unknown, \
1999 .reg = name, \
2000 .visibility = mte_visibility, \
2001 }
2002
el2_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)2003 static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
2004 const struct sys_reg_desc *rd)
2005 {
2006 if (vcpu_has_nv(vcpu))
2007 return 0;
2008
2009 return REG_HIDDEN;
2010 }
2011
bad_vncr_trap(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)2012 static bool bad_vncr_trap(struct kvm_vcpu *vcpu,
2013 struct sys_reg_params *p,
2014 const struct sys_reg_desc *r)
2015 {
2016 /*
2017 * We really shouldn't be here, and this is likely the result
2018 * of a misconfigured trap, as this register should target the
2019 * VNCR page, and nothing else.
2020 */
2021 return bad_trap(vcpu, p, r,
2022 "trap of VNCR-backed register");
2023 }
2024
bad_redir_trap(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)2025 static bool bad_redir_trap(struct kvm_vcpu *vcpu,
2026 struct sys_reg_params *p,
2027 const struct sys_reg_desc *r)
2028 {
2029 /*
2030 * We really shouldn't be here, and this is likely the result
2031 * of a misconfigured trap, as this register should target the
2032 * corresponding EL1, and nothing else.
2033 */
2034 return bad_trap(vcpu, p, r,
2035 "trap of EL2 register redirected to EL1");
2036 }
2037
2038 #define EL2_REG(name, acc, rst, v) { \
2039 SYS_DESC(SYS_##name), \
2040 .access = acc, \
2041 .reset = rst, \
2042 .reg = name, \
2043 .visibility = el2_visibility, \
2044 .val = v, \
2045 }
2046
2047 #define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
2048 #define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2049
2050 /*
2051 * EL{0,1}2 registers are the EL2 view on an EL0 or EL1 register when
2052 * HCR_EL2.E2H==1, and only in the sysreg table for convenience of
2053 * handling traps. Given that, they are always hidden from userspace.
2054 */
hidden_user_visibility(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd)2055 static unsigned int hidden_user_visibility(const struct kvm_vcpu *vcpu,
2056 const struct sys_reg_desc *rd)
2057 {
2058 return REG_HIDDEN_USER;
2059 }
2060
2061 #define EL12_REG(name, acc, rst, v) { \
2062 SYS_DESC(SYS_##name##_EL12), \
2063 .access = acc, \
2064 .reset = rst, \
2065 .reg = name##_EL1, \
2066 .val = v, \
2067 .visibility = hidden_user_visibility, \
2068 }
2069
2070 /*
2071 * Since reset() callback and field val are not used for idregs, they will be
2072 * used for specific purposes for idregs.
2073 * The reset() would return KVM sanitised register value. The value would be the
2074 * same as the host kernel sanitised value if there is no KVM sanitisation.
2075 * The val would be used as a mask indicating writable fields for the idreg.
2076 * Only bits with 1 are writable from userspace. This mask might not be
2077 * necessary in the future whenever all ID registers are enabled as writable
2078 * from userspace.
2079 */
2080
2081 #define ID_DESC(name) \
2082 SYS_DESC(SYS_##name), \
2083 .access = access_id_reg, \
2084 .get_user = get_id_reg \
2085
2086 /* sys_reg_desc initialiser for known cpufeature ID registers */
2087 #define ID_SANITISED(name) { \
2088 ID_DESC(name), \
2089 .set_user = set_id_reg, \
2090 .visibility = id_visibility, \
2091 .reset = kvm_read_sanitised_id_reg, \
2092 .val = 0, \
2093 }
2094
2095 /* sys_reg_desc initialiser for known cpufeature ID registers */
2096 #define AA32_ID_SANITISED(name) { \
2097 ID_DESC(name), \
2098 .set_user = set_id_reg, \
2099 .visibility = aa32_id_visibility, \
2100 .reset = kvm_read_sanitised_id_reg, \
2101 .val = 0, \
2102 }
2103
2104 /* sys_reg_desc initialiser for writable ID registers */
2105 #define ID_WRITABLE(name, mask) { \
2106 ID_DESC(name), \
2107 .set_user = set_id_reg, \
2108 .visibility = id_visibility, \
2109 .reset = kvm_read_sanitised_id_reg, \
2110 .val = mask, \
2111 }
2112
2113 /*
2114 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
2115 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
2116 * (1 <= crm < 8, 0 <= Op2 < 8).
2117 */
2118 #define ID_UNALLOCATED(crm, op2) { \
2119 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
2120 .access = access_id_reg, \
2121 .get_user = get_id_reg, \
2122 .set_user = set_id_reg, \
2123 .visibility = raz_visibility, \
2124 .reset = kvm_read_sanitised_id_reg, \
2125 .val = 0, \
2126 }
2127
2128 /*
2129 * sys_reg_desc initialiser for known ID registers that we hide from guests.
2130 * For now, these are exposed just like unallocated ID regs: they appear
2131 * RAZ for the guest.
2132 */
2133 #define ID_HIDDEN(name) { \
2134 ID_DESC(name), \
2135 .set_user = set_id_reg, \
2136 .visibility = raz_visibility, \
2137 .reset = kvm_read_sanitised_id_reg, \
2138 .val = 0, \
2139 }
2140
access_sp_el1(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)2141 static bool access_sp_el1(struct kvm_vcpu *vcpu,
2142 struct sys_reg_params *p,
2143 const struct sys_reg_desc *r)
2144 {
2145 if (p->is_write)
2146 __vcpu_sys_reg(vcpu, SP_EL1) = p->regval;
2147 else
2148 p->regval = __vcpu_sys_reg(vcpu, SP_EL1);
2149
2150 return true;
2151 }
2152
access_elr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)2153 static bool access_elr(struct kvm_vcpu *vcpu,
2154 struct sys_reg_params *p,
2155 const struct sys_reg_desc *r)
2156 {
2157 if (p->is_write)
2158 vcpu_write_sys_reg(vcpu, p->regval, ELR_EL1);
2159 else
2160 p->regval = vcpu_read_sys_reg(vcpu, ELR_EL1);
2161
2162 return true;
2163 }
2164
access_spsr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)2165 static bool access_spsr(struct kvm_vcpu *vcpu,
2166 struct sys_reg_params *p,
2167 const struct sys_reg_desc *r)
2168 {
2169 if (p->is_write)
2170 __vcpu_sys_reg(vcpu, SPSR_EL1) = p->regval;
2171 else
2172 p->regval = __vcpu_sys_reg(vcpu, SPSR_EL1);
2173
2174 return true;
2175 }
2176
2177 /*
2178 * Architected system registers.
2179 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
2180 *
2181 * Debug handling: We do trap most, if not all debug related system
2182 * registers. The implementation is good enough to ensure that a guest
2183 * can use these with minimal performance degradation. The drawback is
2184 * that we don't implement any of the external debug architecture.
2185 * This should be revisited if we ever encounter a more demanding
2186 * guest...
2187 */
2188 static const struct sys_reg_desc sys_reg_descs[] = {
2189 { SYS_DESC(SYS_DC_ISW), access_dcsw },
2190 { SYS_DESC(SYS_DC_IGSW), access_dcgsw },
2191 { SYS_DESC(SYS_DC_IGDSW), access_dcgsw },
2192 { SYS_DESC(SYS_DC_CSW), access_dcsw },
2193 { SYS_DESC(SYS_DC_CGSW), access_dcgsw },
2194 { SYS_DESC(SYS_DC_CGDSW), access_dcgsw },
2195 { SYS_DESC(SYS_DC_CISW), access_dcsw },
2196 { SYS_DESC(SYS_DC_CIGSW), access_dcgsw },
2197 { SYS_DESC(SYS_DC_CIGDSW), access_dcgsw },
2198
2199 DBG_BCR_BVR_WCR_WVR_EL1(0),
2200 DBG_BCR_BVR_WCR_WVR_EL1(1),
2201 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
2202 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
2203 DBG_BCR_BVR_WCR_WVR_EL1(2),
2204 DBG_BCR_BVR_WCR_WVR_EL1(3),
2205 DBG_BCR_BVR_WCR_WVR_EL1(4),
2206 DBG_BCR_BVR_WCR_WVR_EL1(5),
2207 DBG_BCR_BVR_WCR_WVR_EL1(6),
2208 DBG_BCR_BVR_WCR_WVR_EL1(7),
2209 DBG_BCR_BVR_WCR_WVR_EL1(8),
2210 DBG_BCR_BVR_WCR_WVR_EL1(9),
2211 DBG_BCR_BVR_WCR_WVR_EL1(10),
2212 DBG_BCR_BVR_WCR_WVR_EL1(11),
2213 DBG_BCR_BVR_WCR_WVR_EL1(12),
2214 DBG_BCR_BVR_WCR_WVR_EL1(13),
2215 DBG_BCR_BVR_WCR_WVR_EL1(14),
2216 DBG_BCR_BVR_WCR_WVR_EL1(15),
2217
2218 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
2219 { SYS_DESC(SYS_OSLAR_EL1), trap_oslar_el1 },
2220 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1, reset_val, OSLSR_EL1,
2221 OSLSR_EL1_OSLM_IMPLEMENTED, .set_user = set_oslsr_el1, },
2222 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
2223 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
2224 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
2225 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
2226 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
2227
2228 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
2229 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
2230 // DBGDTR[TR]X_EL0 share the same encoding
2231 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
2232
2233 { SYS_DESC(SYS_DBGVCR32_EL2), trap_undef, reset_val, DBGVCR32_EL2, 0 },
2234
2235 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
2236
2237 /*
2238 * ID regs: all ID_SANITISED() entries here must have corresponding
2239 * entries in arm64_ftr_regs[].
2240 */
2241
2242 /* AArch64 mappings of the AArch32 ID registers */
2243 /* CRm=1 */
2244 AA32_ID_SANITISED(ID_PFR0_EL1),
2245 AA32_ID_SANITISED(ID_PFR1_EL1),
2246 { SYS_DESC(SYS_ID_DFR0_EL1),
2247 .access = access_id_reg,
2248 .get_user = get_id_reg,
2249 .set_user = set_id_dfr0_el1,
2250 .visibility = aa32_id_visibility,
2251 .reset = read_sanitised_id_dfr0_el1,
2252 .val = ID_DFR0_EL1_PerfMon_MASK |
2253 ID_DFR0_EL1_CopDbg_MASK, },
2254 ID_HIDDEN(ID_AFR0_EL1),
2255 AA32_ID_SANITISED(ID_MMFR0_EL1),
2256 AA32_ID_SANITISED(ID_MMFR1_EL1),
2257 AA32_ID_SANITISED(ID_MMFR2_EL1),
2258 AA32_ID_SANITISED(ID_MMFR3_EL1),
2259
2260 /* CRm=2 */
2261 AA32_ID_SANITISED(ID_ISAR0_EL1),
2262 AA32_ID_SANITISED(ID_ISAR1_EL1),
2263 AA32_ID_SANITISED(ID_ISAR2_EL1),
2264 AA32_ID_SANITISED(ID_ISAR3_EL1),
2265 AA32_ID_SANITISED(ID_ISAR4_EL1),
2266 AA32_ID_SANITISED(ID_ISAR5_EL1),
2267 AA32_ID_SANITISED(ID_MMFR4_EL1),
2268 AA32_ID_SANITISED(ID_ISAR6_EL1),
2269
2270 /* CRm=3 */
2271 AA32_ID_SANITISED(MVFR0_EL1),
2272 AA32_ID_SANITISED(MVFR1_EL1),
2273 AA32_ID_SANITISED(MVFR2_EL1),
2274 ID_UNALLOCATED(3,3),
2275 AA32_ID_SANITISED(ID_PFR2_EL1),
2276 ID_HIDDEN(ID_DFR1_EL1),
2277 AA32_ID_SANITISED(ID_MMFR5_EL1),
2278 ID_UNALLOCATED(3,7),
2279
2280 /* AArch64 ID registers */
2281 /* CRm=4 */
2282 { SYS_DESC(SYS_ID_AA64PFR0_EL1),
2283 .access = access_id_reg,
2284 .get_user = get_id_reg,
2285 .set_user = set_id_reg,
2286 .reset = read_sanitised_id_aa64pfr0_el1,
2287 .val = ~(ID_AA64PFR0_EL1_AMU |
2288 ID_AA64PFR0_EL1_MPAM |
2289 ID_AA64PFR0_EL1_SVE |
2290 ID_AA64PFR0_EL1_RAS |
2291 ID_AA64PFR0_EL1_GIC |
2292 ID_AA64PFR0_EL1_AdvSIMD |
2293 ID_AA64PFR0_EL1_FP), },
2294 ID_SANITISED(ID_AA64PFR1_EL1),
2295 ID_UNALLOCATED(4,2),
2296 ID_UNALLOCATED(4,3),
2297 ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0),
2298 ID_HIDDEN(ID_AA64SMFR0_EL1),
2299 ID_UNALLOCATED(4,6),
2300 ID_UNALLOCATED(4,7),
2301
2302 /* CRm=5 */
2303 { SYS_DESC(SYS_ID_AA64DFR0_EL1),
2304 .access = access_id_reg,
2305 .get_user = get_id_reg,
2306 .set_user = set_id_aa64dfr0_el1,
2307 .reset = read_sanitised_id_aa64dfr0_el1,
2308 .val = ID_AA64DFR0_EL1_PMUVer_MASK |
2309 ID_AA64DFR0_EL1_DebugVer_MASK, },
2310 ID_SANITISED(ID_AA64DFR1_EL1),
2311 ID_UNALLOCATED(5,2),
2312 ID_UNALLOCATED(5,3),
2313 ID_HIDDEN(ID_AA64AFR0_EL1),
2314 ID_HIDDEN(ID_AA64AFR1_EL1),
2315 ID_UNALLOCATED(5,6),
2316 ID_UNALLOCATED(5,7),
2317
2318 /* CRm=6 */
2319 ID_WRITABLE(ID_AA64ISAR0_EL1, ~ID_AA64ISAR0_EL1_RES0),
2320 ID_WRITABLE(ID_AA64ISAR1_EL1, ~(ID_AA64ISAR1_EL1_GPI |
2321 ID_AA64ISAR1_EL1_GPA |
2322 ID_AA64ISAR1_EL1_API |
2323 ID_AA64ISAR1_EL1_APA)),
2324 ID_WRITABLE(ID_AA64ISAR2_EL1, ~(ID_AA64ISAR2_EL1_RES0 |
2325 ID_AA64ISAR2_EL1_APA3 |
2326 ID_AA64ISAR2_EL1_GPA3)),
2327 ID_UNALLOCATED(6,3),
2328 ID_UNALLOCATED(6,4),
2329 ID_UNALLOCATED(6,5),
2330 ID_UNALLOCATED(6,6),
2331 ID_UNALLOCATED(6,7),
2332
2333 /* CRm=7 */
2334 ID_WRITABLE(ID_AA64MMFR0_EL1, ~(ID_AA64MMFR0_EL1_RES0 |
2335 ID_AA64MMFR0_EL1_TGRAN4_2 |
2336 ID_AA64MMFR0_EL1_TGRAN64_2 |
2337 ID_AA64MMFR0_EL1_TGRAN16_2)),
2338 ID_WRITABLE(ID_AA64MMFR1_EL1, ~(ID_AA64MMFR1_EL1_RES0 |
2339 ID_AA64MMFR1_EL1_HCX |
2340 ID_AA64MMFR1_EL1_XNX |
2341 ID_AA64MMFR1_EL1_TWED |
2342 ID_AA64MMFR1_EL1_XNX |
2343 ID_AA64MMFR1_EL1_VH |
2344 ID_AA64MMFR1_EL1_VMIDBits)),
2345 ID_WRITABLE(ID_AA64MMFR2_EL1, ~(ID_AA64MMFR2_EL1_RES0 |
2346 ID_AA64MMFR2_EL1_EVT |
2347 ID_AA64MMFR2_EL1_FWB |
2348 ID_AA64MMFR2_EL1_IDS |
2349 ID_AA64MMFR2_EL1_NV |
2350 ID_AA64MMFR2_EL1_CCIDX)),
2351 ID_SANITISED(ID_AA64MMFR3_EL1),
2352 ID_UNALLOCATED(7,4),
2353 ID_UNALLOCATED(7,5),
2354 ID_UNALLOCATED(7,6),
2355 ID_UNALLOCATED(7,7),
2356
2357 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
2358 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
2359 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
2360
2361 MTE_REG(RGSR_EL1),
2362 MTE_REG(GCR_EL1),
2363
2364 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
2365 { SYS_DESC(SYS_TRFCR_EL1), undef_access },
2366 { SYS_DESC(SYS_SMPRI_EL1), undef_access },
2367 { SYS_DESC(SYS_SMCR_EL1), undef_access },
2368 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
2369 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
2370 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
2371 { SYS_DESC(SYS_TCR2_EL1), access_vm_reg, reset_val, TCR2_EL1, 0 },
2372
2373 PTRAUTH_KEY(APIA),
2374 PTRAUTH_KEY(APIB),
2375 PTRAUTH_KEY(APDA),
2376 PTRAUTH_KEY(APDB),
2377 PTRAUTH_KEY(APGA),
2378
2379 { SYS_DESC(SYS_SPSR_EL1), access_spsr},
2380 { SYS_DESC(SYS_ELR_EL1), access_elr},
2381
2382 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
2383 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
2384 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
2385
2386 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
2387 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
2388 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
2389 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
2390 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
2391 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
2392 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
2393 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
2394
2395 MTE_REG(TFSR_EL1),
2396 MTE_REG(TFSRE0_EL1),
2397
2398 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
2399 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
2400
2401 { SYS_DESC(SYS_PMSCR_EL1), undef_access },
2402 { SYS_DESC(SYS_PMSNEVFR_EL1), undef_access },
2403 { SYS_DESC(SYS_PMSICR_EL1), undef_access },
2404 { SYS_DESC(SYS_PMSIRR_EL1), undef_access },
2405 { SYS_DESC(SYS_PMSFCR_EL1), undef_access },
2406 { SYS_DESC(SYS_PMSEVFR_EL1), undef_access },
2407 { SYS_DESC(SYS_PMSLATFR_EL1), undef_access },
2408 { SYS_DESC(SYS_PMSIDR_EL1), undef_access },
2409 { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access },
2410 { SYS_DESC(SYS_PMBPTR_EL1), undef_access },
2411 { SYS_DESC(SYS_PMBSR_EL1), undef_access },
2412 /* PMBIDR_EL1 is not trapped */
2413
2414 { PMU_SYS_REG(PMINTENSET_EL1),
2415 .access = access_pminten, .reg = PMINTENSET_EL1,
2416 .get_user = get_pmreg, .set_user = set_pmreg },
2417 { PMU_SYS_REG(PMINTENCLR_EL1),
2418 .access = access_pminten, .reg = PMINTENSET_EL1,
2419 .get_user = get_pmreg, .set_user = set_pmreg },
2420 { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
2421
2422 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
2423 { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1 },
2424 { SYS_DESC(SYS_PIR_EL1), NULL, reset_unknown, PIR_EL1 },
2425 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
2426
2427 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
2428 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
2429 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
2430 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
2431 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
2432
2433 { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 },
2434 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
2435
2436 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
2437 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
2438 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
2439 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
2440 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
2441 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
2442 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
2443 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
2444 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
2445 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
2446 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
2447 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
2448
2449 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
2450 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
2451
2452 { SYS_DESC(SYS_ACCDATA_EL1), undef_access },
2453
2454 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
2455
2456 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
2457
2458 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
2459 { SYS_DESC(SYS_CLIDR_EL1), access_clidr, reset_clidr, CLIDR_EL1,
2460 .set_user = set_clidr },
2461 { SYS_DESC(SYS_CCSIDR2_EL1), undef_access },
2462 { SYS_DESC(SYS_SMIDR_EL1), undef_access },
2463 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
2464 { SYS_DESC(SYS_CTR_EL0), access_ctr },
2465 { SYS_DESC(SYS_SVCR), undef_access },
2466
2467 { PMU_SYS_REG(PMCR_EL0), .access = access_pmcr, .reset = reset_pmcr,
2468 .reg = PMCR_EL0, .get_user = get_pmcr, .set_user = set_pmcr },
2469 { PMU_SYS_REG(PMCNTENSET_EL0),
2470 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2471 .get_user = get_pmreg, .set_user = set_pmreg },
2472 { PMU_SYS_REG(PMCNTENCLR_EL0),
2473 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2474 .get_user = get_pmreg, .set_user = set_pmreg },
2475 { PMU_SYS_REG(PMOVSCLR_EL0),
2476 .access = access_pmovs, .reg = PMOVSSET_EL0,
2477 .get_user = get_pmreg, .set_user = set_pmreg },
2478 /*
2479 * PM_SWINC_EL0 is exposed to userspace as RAZ/WI, as it was
2480 * previously (and pointlessly) advertised in the past...
2481 */
2482 { PMU_SYS_REG(PMSWINC_EL0),
2483 .get_user = get_raz_reg, .set_user = set_wi_reg,
2484 .access = access_pmswinc, .reset = NULL },
2485 { PMU_SYS_REG(PMSELR_EL0),
2486 .access = access_pmselr, .reset = reset_pmselr, .reg = PMSELR_EL0 },
2487 { PMU_SYS_REG(PMCEID0_EL0),
2488 .access = access_pmceid, .reset = NULL },
2489 { PMU_SYS_REG(PMCEID1_EL0),
2490 .access = access_pmceid, .reset = NULL },
2491 { PMU_SYS_REG(PMCCNTR_EL0),
2492 .access = access_pmu_evcntr, .reset = reset_unknown,
2493 .reg = PMCCNTR_EL0, .get_user = get_pmu_evcntr},
2494 { PMU_SYS_REG(PMXEVTYPER_EL0),
2495 .access = access_pmu_evtyper, .reset = NULL },
2496 { PMU_SYS_REG(PMXEVCNTR_EL0),
2497 .access = access_pmu_evcntr, .reset = NULL },
2498 /*
2499 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
2500 * in 32bit mode. Here we choose to reset it as zero for consistency.
2501 */
2502 { PMU_SYS_REG(PMUSERENR_EL0), .access = access_pmuserenr,
2503 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
2504 { PMU_SYS_REG(PMOVSSET_EL0),
2505 .access = access_pmovs, .reg = PMOVSSET_EL0,
2506 .get_user = get_pmreg, .set_user = set_pmreg },
2507
2508 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
2509 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
2510 { SYS_DESC(SYS_TPIDR2_EL0), undef_access },
2511
2512 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
2513
2514 { SYS_DESC(SYS_AMCR_EL0), undef_access },
2515 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
2516 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
2517 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
2518 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
2519 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
2520 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
2521 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
2522 AMU_AMEVCNTR0_EL0(0),
2523 AMU_AMEVCNTR0_EL0(1),
2524 AMU_AMEVCNTR0_EL0(2),
2525 AMU_AMEVCNTR0_EL0(3),
2526 AMU_AMEVCNTR0_EL0(4),
2527 AMU_AMEVCNTR0_EL0(5),
2528 AMU_AMEVCNTR0_EL0(6),
2529 AMU_AMEVCNTR0_EL0(7),
2530 AMU_AMEVCNTR0_EL0(8),
2531 AMU_AMEVCNTR0_EL0(9),
2532 AMU_AMEVCNTR0_EL0(10),
2533 AMU_AMEVCNTR0_EL0(11),
2534 AMU_AMEVCNTR0_EL0(12),
2535 AMU_AMEVCNTR0_EL0(13),
2536 AMU_AMEVCNTR0_EL0(14),
2537 AMU_AMEVCNTR0_EL0(15),
2538 AMU_AMEVTYPER0_EL0(0),
2539 AMU_AMEVTYPER0_EL0(1),
2540 AMU_AMEVTYPER0_EL0(2),
2541 AMU_AMEVTYPER0_EL0(3),
2542 AMU_AMEVTYPER0_EL0(4),
2543 AMU_AMEVTYPER0_EL0(5),
2544 AMU_AMEVTYPER0_EL0(6),
2545 AMU_AMEVTYPER0_EL0(7),
2546 AMU_AMEVTYPER0_EL0(8),
2547 AMU_AMEVTYPER0_EL0(9),
2548 AMU_AMEVTYPER0_EL0(10),
2549 AMU_AMEVTYPER0_EL0(11),
2550 AMU_AMEVTYPER0_EL0(12),
2551 AMU_AMEVTYPER0_EL0(13),
2552 AMU_AMEVTYPER0_EL0(14),
2553 AMU_AMEVTYPER0_EL0(15),
2554 AMU_AMEVCNTR1_EL0(0),
2555 AMU_AMEVCNTR1_EL0(1),
2556 AMU_AMEVCNTR1_EL0(2),
2557 AMU_AMEVCNTR1_EL0(3),
2558 AMU_AMEVCNTR1_EL0(4),
2559 AMU_AMEVCNTR1_EL0(5),
2560 AMU_AMEVCNTR1_EL0(6),
2561 AMU_AMEVCNTR1_EL0(7),
2562 AMU_AMEVCNTR1_EL0(8),
2563 AMU_AMEVCNTR1_EL0(9),
2564 AMU_AMEVCNTR1_EL0(10),
2565 AMU_AMEVCNTR1_EL0(11),
2566 AMU_AMEVCNTR1_EL0(12),
2567 AMU_AMEVCNTR1_EL0(13),
2568 AMU_AMEVCNTR1_EL0(14),
2569 AMU_AMEVCNTR1_EL0(15),
2570 AMU_AMEVTYPER1_EL0(0),
2571 AMU_AMEVTYPER1_EL0(1),
2572 AMU_AMEVTYPER1_EL0(2),
2573 AMU_AMEVTYPER1_EL0(3),
2574 AMU_AMEVTYPER1_EL0(4),
2575 AMU_AMEVTYPER1_EL0(5),
2576 AMU_AMEVTYPER1_EL0(6),
2577 AMU_AMEVTYPER1_EL0(7),
2578 AMU_AMEVTYPER1_EL0(8),
2579 AMU_AMEVTYPER1_EL0(9),
2580 AMU_AMEVTYPER1_EL0(10),
2581 AMU_AMEVTYPER1_EL0(11),
2582 AMU_AMEVTYPER1_EL0(12),
2583 AMU_AMEVTYPER1_EL0(13),
2584 AMU_AMEVTYPER1_EL0(14),
2585 AMU_AMEVTYPER1_EL0(15),
2586
2587 { SYS_DESC(SYS_CNTPCT_EL0), access_arch_timer },
2588 { SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },
2589 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
2590 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
2591 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
2592
2593 /* PMEVCNTRn_EL0 */
2594 PMU_PMEVCNTR_EL0(0),
2595 PMU_PMEVCNTR_EL0(1),
2596 PMU_PMEVCNTR_EL0(2),
2597 PMU_PMEVCNTR_EL0(3),
2598 PMU_PMEVCNTR_EL0(4),
2599 PMU_PMEVCNTR_EL0(5),
2600 PMU_PMEVCNTR_EL0(6),
2601 PMU_PMEVCNTR_EL0(7),
2602 PMU_PMEVCNTR_EL0(8),
2603 PMU_PMEVCNTR_EL0(9),
2604 PMU_PMEVCNTR_EL0(10),
2605 PMU_PMEVCNTR_EL0(11),
2606 PMU_PMEVCNTR_EL0(12),
2607 PMU_PMEVCNTR_EL0(13),
2608 PMU_PMEVCNTR_EL0(14),
2609 PMU_PMEVCNTR_EL0(15),
2610 PMU_PMEVCNTR_EL0(16),
2611 PMU_PMEVCNTR_EL0(17),
2612 PMU_PMEVCNTR_EL0(18),
2613 PMU_PMEVCNTR_EL0(19),
2614 PMU_PMEVCNTR_EL0(20),
2615 PMU_PMEVCNTR_EL0(21),
2616 PMU_PMEVCNTR_EL0(22),
2617 PMU_PMEVCNTR_EL0(23),
2618 PMU_PMEVCNTR_EL0(24),
2619 PMU_PMEVCNTR_EL0(25),
2620 PMU_PMEVCNTR_EL0(26),
2621 PMU_PMEVCNTR_EL0(27),
2622 PMU_PMEVCNTR_EL0(28),
2623 PMU_PMEVCNTR_EL0(29),
2624 PMU_PMEVCNTR_EL0(30),
2625 /* PMEVTYPERn_EL0 */
2626 PMU_PMEVTYPER_EL0(0),
2627 PMU_PMEVTYPER_EL0(1),
2628 PMU_PMEVTYPER_EL0(2),
2629 PMU_PMEVTYPER_EL0(3),
2630 PMU_PMEVTYPER_EL0(4),
2631 PMU_PMEVTYPER_EL0(5),
2632 PMU_PMEVTYPER_EL0(6),
2633 PMU_PMEVTYPER_EL0(7),
2634 PMU_PMEVTYPER_EL0(8),
2635 PMU_PMEVTYPER_EL0(9),
2636 PMU_PMEVTYPER_EL0(10),
2637 PMU_PMEVTYPER_EL0(11),
2638 PMU_PMEVTYPER_EL0(12),
2639 PMU_PMEVTYPER_EL0(13),
2640 PMU_PMEVTYPER_EL0(14),
2641 PMU_PMEVTYPER_EL0(15),
2642 PMU_PMEVTYPER_EL0(16),
2643 PMU_PMEVTYPER_EL0(17),
2644 PMU_PMEVTYPER_EL0(18),
2645 PMU_PMEVTYPER_EL0(19),
2646 PMU_PMEVTYPER_EL0(20),
2647 PMU_PMEVTYPER_EL0(21),
2648 PMU_PMEVTYPER_EL0(22),
2649 PMU_PMEVTYPER_EL0(23),
2650 PMU_PMEVTYPER_EL0(24),
2651 PMU_PMEVTYPER_EL0(25),
2652 PMU_PMEVTYPER_EL0(26),
2653 PMU_PMEVTYPER_EL0(27),
2654 PMU_PMEVTYPER_EL0(28),
2655 PMU_PMEVTYPER_EL0(29),
2656 PMU_PMEVTYPER_EL0(30),
2657 /*
2658 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
2659 * in 32bit mode. Here we choose to reset it as zero for consistency.
2660 */
2661 { PMU_SYS_REG(PMCCFILTR_EL0), .access = access_pmu_evtyper,
2662 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
2663
2664 EL2_REG_VNCR(VPIDR_EL2, reset_unknown, 0),
2665 EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
2666 EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
2667 EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
2668 EL2_REG_VNCR(HCR_EL2, reset_val, 0),
2669 EL2_REG(MDCR_EL2, access_rw, reset_val, 0),
2670 EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
2671 EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
2672 EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
2673 EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
2674 EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
2675 EL2_REG_VNCR(HACR_EL2, reset_val, 0),
2676
2677 EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
2678
2679 EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
2680 EL2_REG(TTBR1_EL2, access_rw, reset_val, 0),
2681 EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1),
2682 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
2683 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
2684
2685 { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 },
2686 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
2687 EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
2688 EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
2689 EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
2690 EL2_REG_REDIR(ELR_EL2, reset_val, 0),
2691 { SYS_DESC(SYS_SP_EL1), access_sp_el1},
2692
2693 /* AArch32 SPSR_* are RES0 if trapped from a NV guest */
2694 { SYS_DESC(SYS_SPSR_irq), .access = trap_raz_wi,
2695 .visibility = hidden_user_visibility },
2696 { SYS_DESC(SYS_SPSR_abt), .access = trap_raz_wi,
2697 .visibility = hidden_user_visibility },
2698 { SYS_DESC(SYS_SPSR_und), .access = trap_raz_wi,
2699 .visibility = hidden_user_visibility },
2700 { SYS_DESC(SYS_SPSR_fiq), .access = trap_raz_wi,
2701 .visibility = hidden_user_visibility },
2702
2703 { SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 },
2704 EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
2705 EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
2706 EL2_REG_REDIR(ESR_EL2, reset_val, 0),
2707 { SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 },
2708
2709 EL2_REG_REDIR(FAR_EL2, reset_val, 0),
2710 EL2_REG(HPFAR_EL2, access_rw, reset_val, 0),
2711
2712 EL2_REG(MAIR_EL2, access_rw, reset_val, 0),
2713 EL2_REG(AMAIR_EL2, access_rw, reset_val, 0),
2714
2715 EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
2716 EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
2717 { SYS_DESC(SYS_RMR_EL2), trap_undef },
2718
2719 EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
2720 EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
2721
2722 EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
2723 EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
2724
2725 EL12_REG(CNTKCTL, access_rw, reset_val, 0),
2726
2727 EL2_REG(SP_EL2, NULL, reset_unknown, 0),
2728 };
2729
2730 static const struct sys_reg_desc *first_idreg;
2731
trap_dbgdidr(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)2732 static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
2733 struct sys_reg_params *p,
2734 const struct sys_reg_desc *r)
2735 {
2736 if (p->is_write) {
2737 return ignore_write(vcpu, p);
2738 } else {
2739 u64 dfr = IDREG(vcpu->kvm, SYS_ID_AA64DFR0_EL1);
2740 u64 pfr = IDREG(vcpu->kvm, SYS_ID_AA64PFR0_EL1);
2741 u32 el3 = !!SYS_FIELD_GET(ID_AA64PFR0_EL1, EL3, pfr);
2742
2743 p->regval = ((SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr) << 28) |
2744 (SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr) << 24) |
2745 (SYS_FIELD_GET(ID_AA64DFR0_EL1, CTX_CMPs, dfr) << 20) |
2746 (SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, dfr) << 16) |
2747 (1 << 15) | (el3 << 14) | (el3 << 12));
2748 return true;
2749 }
2750 }
2751
2752 /*
2753 * AArch32 debug register mappings
2754 *
2755 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
2756 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
2757 *
2758 * None of the other registers share their location, so treat them as
2759 * if they were 64bit.
2760 */
2761 #define DBG_BCR_BVR_WCR_WVR(n) \
2762 /* DBGBVRn */ \
2763 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
2764 /* DBGBCRn */ \
2765 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
2766 /* DBGWVRn */ \
2767 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
2768 /* DBGWCRn */ \
2769 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
2770
2771 #define DBGBXVR(n) \
2772 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
2773
2774 /*
2775 * Trapped cp14 registers. We generally ignore most of the external
2776 * debug, on the principle that they don't really make sense to a
2777 * guest. Revisit this one day, would this principle change.
2778 */
2779 static const struct sys_reg_desc cp14_regs[] = {
2780 /* DBGDIDR */
2781 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
2782 /* DBGDTRRXext */
2783 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
2784
2785 DBG_BCR_BVR_WCR_WVR(0),
2786 /* DBGDSCRint */
2787 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
2788 DBG_BCR_BVR_WCR_WVR(1),
2789 /* DBGDCCINT */
2790 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
2791 /* DBGDSCRext */
2792 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
2793 DBG_BCR_BVR_WCR_WVR(2),
2794 /* DBGDTR[RT]Xint */
2795 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
2796 /* DBGDTR[RT]Xext */
2797 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
2798 DBG_BCR_BVR_WCR_WVR(3),
2799 DBG_BCR_BVR_WCR_WVR(4),
2800 DBG_BCR_BVR_WCR_WVR(5),
2801 /* DBGWFAR */
2802 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
2803 /* DBGOSECCR */
2804 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
2805 DBG_BCR_BVR_WCR_WVR(6),
2806 /* DBGVCR */
2807 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
2808 DBG_BCR_BVR_WCR_WVR(7),
2809 DBG_BCR_BVR_WCR_WVR(8),
2810 DBG_BCR_BVR_WCR_WVR(9),
2811 DBG_BCR_BVR_WCR_WVR(10),
2812 DBG_BCR_BVR_WCR_WVR(11),
2813 DBG_BCR_BVR_WCR_WVR(12),
2814 DBG_BCR_BVR_WCR_WVR(13),
2815 DBG_BCR_BVR_WCR_WVR(14),
2816 DBG_BCR_BVR_WCR_WVR(15),
2817
2818 /* DBGDRAR (32bit) */
2819 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
2820
2821 DBGBXVR(0),
2822 /* DBGOSLAR */
2823 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_oslar_el1 },
2824 DBGBXVR(1),
2825 /* DBGOSLSR */
2826 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1, NULL, OSLSR_EL1 },
2827 DBGBXVR(2),
2828 DBGBXVR(3),
2829 /* DBGOSDLR */
2830 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
2831 DBGBXVR(4),
2832 /* DBGPRCR */
2833 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
2834 DBGBXVR(5),
2835 DBGBXVR(6),
2836 DBGBXVR(7),
2837 DBGBXVR(8),
2838 DBGBXVR(9),
2839 DBGBXVR(10),
2840 DBGBXVR(11),
2841 DBGBXVR(12),
2842 DBGBXVR(13),
2843 DBGBXVR(14),
2844 DBGBXVR(15),
2845
2846 /* DBGDSAR (32bit) */
2847 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
2848
2849 /* DBGDEVID2 */
2850 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
2851 /* DBGDEVID1 */
2852 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
2853 /* DBGDEVID */
2854 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
2855 /* DBGCLAIMSET */
2856 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
2857 /* DBGCLAIMCLR */
2858 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
2859 /* DBGAUTHSTATUS */
2860 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
2861 };
2862
2863 /* Trapped cp14 64bit registers */
2864 static const struct sys_reg_desc cp14_64_regs[] = {
2865 /* DBGDRAR (64bit) */
2866 { Op1( 0), CRm( 1), .access = trap_raz_wi },
2867
2868 /* DBGDSAR (64bit) */
2869 { Op1( 0), CRm( 2), .access = trap_raz_wi },
2870 };
2871
2872 #define CP15_PMU_SYS_REG(_map, _Op1, _CRn, _CRm, _Op2) \
2873 AA32(_map), \
2874 Op1(_Op1), CRn(_CRn), CRm(_CRm), Op2(_Op2), \
2875 .visibility = pmu_visibility
2876
2877 /* Macro to expand the PMEVCNTRn register */
2878 #define PMU_PMEVCNTR(n) \
2879 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2880 (0b1000 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2881 .access = access_pmu_evcntr }
2882
2883 /* Macro to expand the PMEVTYPERn register */
2884 #define PMU_PMEVTYPER(n) \
2885 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2886 (0b1100 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2887 .access = access_pmu_evtyper }
2888 /*
2889 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
2890 * depending on the way they are accessed (as a 32bit or a 64bit
2891 * register).
2892 */
2893 static const struct sys_reg_desc cp15_regs[] = {
2894 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
2895 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
2896 /* ACTLR */
2897 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
2898 /* ACTLR2 */
2899 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
2900 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2901 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
2902 /* TTBCR */
2903 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
2904 /* TTBCR2 */
2905 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
2906 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
2907 /* DFSR */
2908 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
2909 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
2910 /* ADFSR */
2911 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
2912 /* AIFSR */
2913 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
2914 /* DFAR */
2915 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
2916 /* IFAR */
2917 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
2918
2919 /*
2920 * DC{C,I,CI}SW operations:
2921 */
2922 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
2923 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
2924 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
2925
2926 /* PMU */
2927 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 0), .access = access_pmcr },
2928 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 1), .access = access_pmcnten },
2929 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 2), .access = access_pmcnten },
2930 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 3), .access = access_pmovs },
2931 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 4), .access = access_pmswinc },
2932 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 5), .access = access_pmselr },
2933 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 6), .access = access_pmceid },
2934 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 7), .access = access_pmceid },
2935 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 0), .access = access_pmu_evcntr },
2936 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 1), .access = access_pmu_evtyper },
2937 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 2), .access = access_pmu_evcntr },
2938 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 0), .access = access_pmuserenr },
2939 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 1), .access = access_pminten },
2940 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 2), .access = access_pminten },
2941 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 3), .access = access_pmovs },
2942 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid },
2943 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid },
2944 /* PMMIR */
2945 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi },
2946
2947 /* PRRR/MAIR0 */
2948 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
2949 /* NMRR/MAIR1 */
2950 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
2951 /* AMAIR0 */
2952 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
2953 /* AMAIR1 */
2954 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
2955
2956 /* ICC_SRE */
2957 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
2958
2959 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
2960
2961 /* Arch Tmers */
2962 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
2963 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
2964
2965 /* PMEVCNTRn */
2966 PMU_PMEVCNTR(0),
2967 PMU_PMEVCNTR(1),
2968 PMU_PMEVCNTR(2),
2969 PMU_PMEVCNTR(3),
2970 PMU_PMEVCNTR(4),
2971 PMU_PMEVCNTR(5),
2972 PMU_PMEVCNTR(6),
2973 PMU_PMEVCNTR(7),
2974 PMU_PMEVCNTR(8),
2975 PMU_PMEVCNTR(9),
2976 PMU_PMEVCNTR(10),
2977 PMU_PMEVCNTR(11),
2978 PMU_PMEVCNTR(12),
2979 PMU_PMEVCNTR(13),
2980 PMU_PMEVCNTR(14),
2981 PMU_PMEVCNTR(15),
2982 PMU_PMEVCNTR(16),
2983 PMU_PMEVCNTR(17),
2984 PMU_PMEVCNTR(18),
2985 PMU_PMEVCNTR(19),
2986 PMU_PMEVCNTR(20),
2987 PMU_PMEVCNTR(21),
2988 PMU_PMEVCNTR(22),
2989 PMU_PMEVCNTR(23),
2990 PMU_PMEVCNTR(24),
2991 PMU_PMEVCNTR(25),
2992 PMU_PMEVCNTR(26),
2993 PMU_PMEVCNTR(27),
2994 PMU_PMEVCNTR(28),
2995 PMU_PMEVCNTR(29),
2996 PMU_PMEVCNTR(30),
2997 /* PMEVTYPERn */
2998 PMU_PMEVTYPER(0),
2999 PMU_PMEVTYPER(1),
3000 PMU_PMEVTYPER(2),
3001 PMU_PMEVTYPER(3),
3002 PMU_PMEVTYPER(4),
3003 PMU_PMEVTYPER(5),
3004 PMU_PMEVTYPER(6),
3005 PMU_PMEVTYPER(7),
3006 PMU_PMEVTYPER(8),
3007 PMU_PMEVTYPER(9),
3008 PMU_PMEVTYPER(10),
3009 PMU_PMEVTYPER(11),
3010 PMU_PMEVTYPER(12),
3011 PMU_PMEVTYPER(13),
3012 PMU_PMEVTYPER(14),
3013 PMU_PMEVTYPER(15),
3014 PMU_PMEVTYPER(16),
3015 PMU_PMEVTYPER(17),
3016 PMU_PMEVTYPER(18),
3017 PMU_PMEVTYPER(19),
3018 PMU_PMEVTYPER(20),
3019 PMU_PMEVTYPER(21),
3020 PMU_PMEVTYPER(22),
3021 PMU_PMEVTYPER(23),
3022 PMU_PMEVTYPER(24),
3023 PMU_PMEVTYPER(25),
3024 PMU_PMEVTYPER(26),
3025 PMU_PMEVTYPER(27),
3026 PMU_PMEVTYPER(28),
3027 PMU_PMEVTYPER(29),
3028 PMU_PMEVTYPER(30),
3029 /* PMCCFILTR */
3030 { CP15_PMU_SYS_REG(DIRECT, 0, 14, 15, 7), .access = access_pmu_evtyper },
3031
3032 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
3033 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
3034
3035 /* CCSIDR2 */
3036 { Op1(1), CRn( 0), CRm( 0), Op2(2), undef_access },
3037
3038 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
3039 };
3040
3041 static const struct sys_reg_desc cp15_64_regs[] = {
3042 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
3043 { CP15_PMU_SYS_REG(DIRECT, 0, 0, 9, 0), .access = access_pmu_evcntr },
3044 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
3045 { SYS_DESC(SYS_AARCH32_CNTPCT), access_arch_timer },
3046 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
3047 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
3048 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
3049 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
3050 { SYS_DESC(SYS_AARCH32_CNTPCTSS), access_arch_timer },
3051 };
3052
check_sysreg_table(const struct sys_reg_desc * table,unsigned int n,bool is_32)3053 static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
3054 bool is_32)
3055 {
3056 unsigned int i;
3057
3058 for (i = 0; i < n; i++) {
3059 if (!is_32 && table[i].reg && !table[i].reset) {
3060 kvm_err("sys_reg table %pS entry %d lacks reset\n", &table[i], i);
3061 return false;
3062 }
3063
3064 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
3065 kvm_err("sys_reg table %pS entry %d out of order\n", &table[i - 1], i - 1);
3066 return false;
3067 }
3068 }
3069
3070 return true;
3071 }
3072
kvm_handle_cp14_load_store(struct kvm_vcpu * vcpu)3073 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
3074 {
3075 kvm_inject_undefined(vcpu);
3076 return 1;
3077 }
3078
perform_access(struct kvm_vcpu * vcpu,struct sys_reg_params * params,const struct sys_reg_desc * r)3079 static void perform_access(struct kvm_vcpu *vcpu,
3080 struct sys_reg_params *params,
3081 const struct sys_reg_desc *r)
3082 {
3083 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
3084
3085 /* Check for regs disabled by runtime config */
3086 if (sysreg_hidden(vcpu, r)) {
3087 kvm_inject_undefined(vcpu);
3088 return;
3089 }
3090
3091 /*
3092 * Not having an accessor means that we have configured a trap
3093 * that we don't know how to handle. This certainly qualifies
3094 * as a gross bug that should be fixed right away.
3095 */
3096 BUG_ON(!r->access);
3097
3098 /* Skip instruction if instructed so */
3099 if (likely(r->access(vcpu, params, r)))
3100 kvm_incr_pc(vcpu);
3101 }
3102
3103 /*
3104 * emulate_cp -- tries to match a sys_reg access in a handling table, and
3105 * call the corresponding trap handler.
3106 *
3107 * @params: pointer to the descriptor of the access
3108 * @table: array of trap descriptors
3109 * @num: size of the trap descriptor array
3110 *
3111 * Return true if the access has been handled, false if not.
3112 */
emulate_cp(struct kvm_vcpu * vcpu,struct sys_reg_params * params,const struct sys_reg_desc * table,size_t num)3113 static bool emulate_cp(struct kvm_vcpu *vcpu,
3114 struct sys_reg_params *params,
3115 const struct sys_reg_desc *table,
3116 size_t num)
3117 {
3118 const struct sys_reg_desc *r;
3119
3120 if (!table)
3121 return false; /* Not handled */
3122
3123 r = find_reg(params, table, num);
3124
3125 if (r) {
3126 perform_access(vcpu, params, r);
3127 return true;
3128 }
3129
3130 /* Not handled */
3131 return false;
3132 }
3133
unhandled_cp_access(struct kvm_vcpu * vcpu,struct sys_reg_params * params)3134 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
3135 struct sys_reg_params *params)
3136 {
3137 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
3138 int cp = -1;
3139
3140 switch (esr_ec) {
3141 case ESR_ELx_EC_CP15_32:
3142 case ESR_ELx_EC_CP15_64:
3143 cp = 15;
3144 break;
3145 case ESR_ELx_EC_CP14_MR:
3146 case ESR_ELx_EC_CP14_64:
3147 cp = 14;
3148 break;
3149 default:
3150 WARN_ON(1);
3151 }
3152
3153 print_sys_reg_msg(params,
3154 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
3155 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3156 kvm_inject_undefined(vcpu);
3157 }
3158
3159 /**
3160 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
3161 * @vcpu: The VCPU pointer
3162 * @run: The kvm_run struct
3163 */
kvm_handle_cp_64(struct kvm_vcpu * vcpu,const struct sys_reg_desc * global,size_t nr_global)3164 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
3165 const struct sys_reg_desc *global,
3166 size_t nr_global)
3167 {
3168 struct sys_reg_params params;
3169 u64 esr = kvm_vcpu_get_esr(vcpu);
3170 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3171 int Rt2 = (esr >> 10) & 0x1f;
3172
3173 params.CRm = (esr >> 1) & 0xf;
3174 params.is_write = ((esr & 1) == 0);
3175
3176 params.Op0 = 0;
3177 params.Op1 = (esr >> 16) & 0xf;
3178 params.Op2 = 0;
3179 params.CRn = 0;
3180
3181 /*
3182 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
3183 * backends between AArch32 and AArch64, we get away with it.
3184 */
3185 if (params.is_write) {
3186 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
3187 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
3188 }
3189
3190 /*
3191 * If the table contains a handler, handle the
3192 * potential register operation in the case of a read and return
3193 * with success.
3194 */
3195 if (emulate_cp(vcpu, ¶ms, global, nr_global)) {
3196 /* Split up the value between registers for the read side */
3197 if (!params.is_write) {
3198 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
3199 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
3200 }
3201
3202 return 1;
3203 }
3204
3205 unhandled_cp_access(vcpu, ¶ms);
3206 return 1;
3207 }
3208
3209 static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params);
3210
3211 /*
3212 * The CP10 ID registers are architecturally mapped to AArch64 feature
3213 * registers. Abuse that fact so we can rely on the AArch64 handler for accesses
3214 * from AArch32.
3215 */
kvm_esr_cp10_id_to_sys64(u64 esr,struct sys_reg_params * params)3216 static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params)
3217 {
3218 u8 reg_id = (esr >> 10) & 0xf;
3219 bool valid;
3220
3221 params->is_write = ((esr & 1) == 0);
3222 params->Op0 = 3;
3223 params->Op1 = 0;
3224 params->CRn = 0;
3225 params->CRm = 3;
3226
3227 /* CP10 ID registers are read-only */
3228 valid = !params->is_write;
3229
3230 switch (reg_id) {
3231 /* MVFR0 */
3232 case 0b0111:
3233 params->Op2 = 0;
3234 break;
3235 /* MVFR1 */
3236 case 0b0110:
3237 params->Op2 = 1;
3238 break;
3239 /* MVFR2 */
3240 case 0b0101:
3241 params->Op2 = 2;
3242 break;
3243 default:
3244 valid = false;
3245 }
3246
3247 if (valid)
3248 return true;
3249
3250 kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
3251 params->is_write ? "write" : "read", reg_id);
3252 return false;
3253 }
3254
3255 /**
3256 * kvm_handle_cp10_id() - Handles a VMRS trap on guest access to a 'Media and
3257 * VFP Register' from AArch32.
3258 * @vcpu: The vCPU pointer
3259 *
3260 * MVFR{0-2} are architecturally mapped to the AArch64 MVFR{0-2}_EL1 registers.
3261 * Work out the correct AArch64 system register encoding and reroute to the
3262 * AArch64 system register emulation.
3263 */
kvm_handle_cp10_id(struct kvm_vcpu * vcpu)3264 int kvm_handle_cp10_id(struct kvm_vcpu *vcpu)
3265 {
3266 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3267 u64 esr = kvm_vcpu_get_esr(vcpu);
3268 struct sys_reg_params params;
3269
3270 /* UNDEF on any unhandled register access */
3271 if (!kvm_esr_cp10_id_to_sys64(esr, ¶ms)) {
3272 kvm_inject_undefined(vcpu);
3273 return 1;
3274 }
3275
3276 if (emulate_sys_reg(vcpu, ¶ms))
3277 vcpu_set_reg(vcpu, Rt, params.regval);
3278
3279 return 1;
3280 }
3281
3282 /**
3283 * kvm_emulate_cp15_id_reg() - Handles an MRC trap on a guest CP15 access where
3284 * CRn=0, which corresponds to the AArch32 feature
3285 * registers.
3286 * @vcpu: the vCPU pointer
3287 * @params: the system register access parameters.
3288 *
3289 * Our cp15 system register tables do not enumerate the AArch32 feature
3290 * registers. Conveniently, our AArch64 table does, and the AArch32 system
3291 * register encoding can be trivially remapped into the AArch64 for the feature
3292 * registers: Append op0=3, leaving op1, CRn, CRm, and op2 the same.
3293 *
3294 * According to DDI0487G.b G7.3.1, paragraph "Behavior of VMSAv8-32 32-bit
3295 * System registers with (coproc=0b1111, CRn==c0)", read accesses from this
3296 * range are either UNKNOWN or RES0. Rerouting remains architectural as we
3297 * treat undefined registers in this range as RAZ.
3298 */
kvm_emulate_cp15_id_reg(struct kvm_vcpu * vcpu,struct sys_reg_params * params)3299 static int kvm_emulate_cp15_id_reg(struct kvm_vcpu *vcpu,
3300 struct sys_reg_params *params)
3301 {
3302 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3303
3304 /* Treat impossible writes to RO registers as UNDEFINED */
3305 if (params->is_write) {
3306 unhandled_cp_access(vcpu, params);
3307 return 1;
3308 }
3309
3310 params->Op0 = 3;
3311
3312 /*
3313 * All registers where CRm > 3 are known to be UNKNOWN/RAZ from AArch32.
3314 * Avoid conflicting with future expansion of AArch64 feature registers
3315 * and simply treat them as RAZ here.
3316 */
3317 if (params->CRm > 3)
3318 params->regval = 0;
3319 else if (!emulate_sys_reg(vcpu, params))
3320 return 1;
3321
3322 vcpu_set_reg(vcpu, Rt, params->regval);
3323 return 1;
3324 }
3325
3326 /**
3327 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
3328 * @vcpu: The VCPU pointer
3329 * @run: The kvm_run struct
3330 */
kvm_handle_cp_32(struct kvm_vcpu * vcpu,struct sys_reg_params * params,const struct sys_reg_desc * global,size_t nr_global)3331 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
3332 struct sys_reg_params *params,
3333 const struct sys_reg_desc *global,
3334 size_t nr_global)
3335 {
3336 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3337
3338 params->regval = vcpu_get_reg(vcpu, Rt);
3339
3340 if (emulate_cp(vcpu, params, global, nr_global)) {
3341 if (!params->is_write)
3342 vcpu_set_reg(vcpu, Rt, params->regval);
3343 return 1;
3344 }
3345
3346 unhandled_cp_access(vcpu, params);
3347 return 1;
3348 }
3349
kvm_handle_cp15_64(struct kvm_vcpu * vcpu)3350 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
3351 {
3352 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
3353 }
3354
kvm_handle_cp15_32(struct kvm_vcpu * vcpu)3355 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
3356 {
3357 struct sys_reg_params params;
3358
3359 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3360
3361 /*
3362 * Certain AArch32 ID registers are handled by rerouting to the AArch64
3363 * system register table. Registers in the ID range where CRm=0 are
3364 * excluded from this scheme as they do not trivially map into AArch64
3365 * system register encodings.
3366 */
3367 if (params.Op1 == 0 && params.CRn == 0 && params.CRm)
3368 return kvm_emulate_cp15_id_reg(vcpu, ¶ms);
3369
3370 return kvm_handle_cp_32(vcpu, ¶ms, cp15_regs, ARRAY_SIZE(cp15_regs));
3371 }
3372
kvm_handle_cp14_64(struct kvm_vcpu * vcpu)3373 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
3374 {
3375 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
3376 }
3377
kvm_handle_cp14_32(struct kvm_vcpu * vcpu)3378 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
3379 {
3380 struct sys_reg_params params;
3381
3382 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3383
3384 return kvm_handle_cp_32(vcpu, ¶ms, cp14_regs, ARRAY_SIZE(cp14_regs));
3385 }
3386
is_imp_def_sys_reg(struct sys_reg_params * params)3387 static bool is_imp_def_sys_reg(struct sys_reg_params *params)
3388 {
3389 // See ARM DDI 0487E.a, section D12.3.2
3390 return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
3391 }
3392
3393 /**
3394 * emulate_sys_reg - Emulate a guest access to an AArch64 system register
3395 * @vcpu: The VCPU pointer
3396 * @params: Decoded system register parameters
3397 *
3398 * Return: true if the system register access was successful, false otherwise.
3399 */
emulate_sys_reg(struct kvm_vcpu * vcpu,struct sys_reg_params * params)3400 static bool emulate_sys_reg(struct kvm_vcpu *vcpu,
3401 struct sys_reg_params *params)
3402 {
3403 const struct sys_reg_desc *r;
3404
3405 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3406
3407 if (likely(r)) {
3408 perform_access(vcpu, params, r);
3409 return true;
3410 }
3411
3412 if (is_imp_def_sys_reg(params)) {
3413 kvm_inject_undefined(vcpu);
3414 } else {
3415 print_sys_reg_msg(params,
3416 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
3417 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3418 kvm_inject_undefined(vcpu);
3419 }
3420 return false;
3421 }
3422
kvm_reset_id_regs(struct kvm_vcpu * vcpu)3423 static void kvm_reset_id_regs(struct kvm_vcpu *vcpu)
3424 {
3425 const struct sys_reg_desc *idreg = first_idreg;
3426 u32 id = reg_to_encoding(idreg);
3427 struct kvm *kvm = vcpu->kvm;
3428
3429 if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
3430 return;
3431
3432 lockdep_assert_held(&kvm->arch.config_lock);
3433
3434 /* Initialize all idregs */
3435 while (is_id_reg(id)) {
3436 IDREG(kvm, id) = idreg->reset(vcpu, idreg);
3437
3438 idreg++;
3439 id = reg_to_encoding(idreg);
3440 }
3441
3442 set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags);
3443 }
3444
3445 /**
3446 * kvm_reset_sys_regs - sets system registers to reset value
3447 * @vcpu: The VCPU pointer
3448 *
3449 * This function finds the right table above and sets the registers on the
3450 * virtual CPU struct to their architecturally defined reset values.
3451 */
kvm_reset_sys_regs(struct kvm_vcpu * vcpu)3452 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
3453 {
3454 unsigned long i;
3455
3456 kvm_reset_id_regs(vcpu);
3457
3458 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3459 const struct sys_reg_desc *r = &sys_reg_descs[i];
3460
3461 if (is_id_reg(reg_to_encoding(r)))
3462 continue;
3463
3464 if (r->reset)
3465 r->reset(vcpu, r);
3466 }
3467 }
3468
3469 /**
3470 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
3471 * @vcpu: The VCPU pointer
3472 */
kvm_handle_sys_reg(struct kvm_vcpu * vcpu)3473 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
3474 {
3475 struct sys_reg_params params;
3476 unsigned long esr = kvm_vcpu_get_esr(vcpu);
3477 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3478
3479 trace_kvm_handle_sys_reg(esr);
3480
3481 if (__check_nv_sr_forward(vcpu))
3482 return 1;
3483
3484 params = esr_sys64_to_params(esr);
3485 params.regval = vcpu_get_reg(vcpu, Rt);
3486
3487 if (!emulate_sys_reg(vcpu, ¶ms))
3488 return 1;
3489
3490 if (!params.is_write)
3491 vcpu_set_reg(vcpu, Rt, params.regval);
3492 return 1;
3493 }
3494
3495 /******************************************************************************
3496 * Userspace API
3497 *****************************************************************************/
3498
index_to_params(u64 id,struct sys_reg_params * params)3499 static bool index_to_params(u64 id, struct sys_reg_params *params)
3500 {
3501 switch (id & KVM_REG_SIZE_MASK) {
3502 case KVM_REG_SIZE_U64:
3503 /* Any unused index bits means it's not valid. */
3504 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
3505 | KVM_REG_ARM_COPROC_MASK
3506 | KVM_REG_ARM64_SYSREG_OP0_MASK
3507 | KVM_REG_ARM64_SYSREG_OP1_MASK
3508 | KVM_REG_ARM64_SYSREG_CRN_MASK
3509 | KVM_REG_ARM64_SYSREG_CRM_MASK
3510 | KVM_REG_ARM64_SYSREG_OP2_MASK))
3511 return false;
3512 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
3513 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
3514 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
3515 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
3516 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
3517 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
3518 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
3519 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
3520 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
3521 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
3522 return true;
3523 default:
3524 return false;
3525 }
3526 }
3527
get_reg_by_id(u64 id,const struct sys_reg_desc table[],unsigned int num)3528 const struct sys_reg_desc *get_reg_by_id(u64 id,
3529 const struct sys_reg_desc table[],
3530 unsigned int num)
3531 {
3532 struct sys_reg_params params;
3533
3534 if (!index_to_params(id, ¶ms))
3535 return NULL;
3536
3537 return find_reg(¶ms, table, num);
3538 }
3539
3540 /* Decode an index value, and find the sys_reg_desc entry. */
3541 static const struct sys_reg_desc *
id_to_sys_reg_desc(struct kvm_vcpu * vcpu,u64 id,const struct sys_reg_desc table[],unsigned int num)3542 id_to_sys_reg_desc(struct kvm_vcpu *vcpu, u64 id,
3543 const struct sys_reg_desc table[], unsigned int num)
3544
3545 {
3546 const struct sys_reg_desc *r;
3547
3548 /* We only do sys_reg for now. */
3549 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
3550 return NULL;
3551
3552 r = get_reg_by_id(id, table, num);
3553
3554 /* Not saved in the sys_reg array and not otherwise accessible? */
3555 if (r && (!(r->reg || r->get_user) || sysreg_hidden(vcpu, r)))
3556 r = NULL;
3557
3558 return r;
3559 }
3560
3561 /*
3562 * These are the invariant sys_reg registers: we let the guest see the
3563 * host versions of these, so they're part of the guest state.
3564 *
3565 * A future CPU may provide a mechanism to present different values to
3566 * the guest, or a future kvm may trap them.
3567 */
3568
3569 #define FUNCTION_INVARIANT(reg) \
3570 static u64 get_##reg(struct kvm_vcpu *v, \
3571 const struct sys_reg_desc *r) \
3572 { \
3573 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
3574 return ((struct sys_reg_desc *)r)->val; \
3575 }
3576
3577 FUNCTION_INVARIANT(midr_el1)
FUNCTION_INVARIANT(revidr_el1)3578 FUNCTION_INVARIANT(revidr_el1)
3579 FUNCTION_INVARIANT(aidr_el1)
3580
3581 static u64 get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
3582 {
3583 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
3584 return ((struct sys_reg_desc *)r)->val;
3585 }
3586
3587 /* ->val is filled in by kvm_sys_reg_table_init() */
3588 static struct sys_reg_desc invariant_sys_regs[] __ro_after_init = {
3589 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
3590 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
3591 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
3592 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
3593 };
3594
get_invariant_sys_reg(u64 id,u64 __user * uaddr)3595 static int get_invariant_sys_reg(u64 id, u64 __user *uaddr)
3596 {
3597 const struct sys_reg_desc *r;
3598
3599 r = get_reg_by_id(id, invariant_sys_regs,
3600 ARRAY_SIZE(invariant_sys_regs));
3601 if (!r)
3602 return -ENOENT;
3603
3604 return put_user(r->val, uaddr);
3605 }
3606
set_invariant_sys_reg(u64 id,u64 __user * uaddr)3607 static int set_invariant_sys_reg(u64 id, u64 __user *uaddr)
3608 {
3609 const struct sys_reg_desc *r;
3610 u64 val;
3611
3612 r = get_reg_by_id(id, invariant_sys_regs,
3613 ARRAY_SIZE(invariant_sys_regs));
3614 if (!r)
3615 return -ENOENT;
3616
3617 if (get_user(val, uaddr))
3618 return -EFAULT;
3619
3620 /* This is what we mean by invariant: you can't change it. */
3621 if (r->val != val)
3622 return -EINVAL;
3623
3624 return 0;
3625 }
3626
demux_c15_get(struct kvm_vcpu * vcpu,u64 id,void __user * uaddr)3627 static int demux_c15_get(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
3628 {
3629 u32 val;
3630 u32 __user *uval = uaddr;
3631
3632 /* Fail if we have unknown bits set. */
3633 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3634 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3635 return -ENOENT;
3636
3637 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3638 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3639 if (KVM_REG_SIZE(id) != 4)
3640 return -ENOENT;
3641 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3642 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
3643 if (val >= CSSELR_MAX)
3644 return -ENOENT;
3645
3646 return put_user(get_ccsidr(vcpu, val), uval);
3647 default:
3648 return -ENOENT;
3649 }
3650 }
3651
demux_c15_set(struct kvm_vcpu * vcpu,u64 id,void __user * uaddr)3652 static int demux_c15_set(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
3653 {
3654 u32 val, newval;
3655 u32 __user *uval = uaddr;
3656
3657 /* Fail if we have unknown bits set. */
3658 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3659 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3660 return -ENOENT;
3661
3662 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3663 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3664 if (KVM_REG_SIZE(id) != 4)
3665 return -ENOENT;
3666 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3667 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
3668 if (val >= CSSELR_MAX)
3669 return -ENOENT;
3670
3671 if (get_user(newval, uval))
3672 return -EFAULT;
3673
3674 return set_ccsidr(vcpu, val, newval);
3675 default:
3676 return -ENOENT;
3677 }
3678 }
3679
kvm_sys_reg_get_user(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg,const struct sys_reg_desc table[],unsigned int num)3680 int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3681 const struct sys_reg_desc table[], unsigned int num)
3682 {
3683 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
3684 const struct sys_reg_desc *r;
3685 u64 val;
3686 int ret;
3687
3688 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
3689 if (!r || sysreg_hidden_user(vcpu, r))
3690 return -ENOENT;
3691
3692 if (r->get_user) {
3693 ret = (r->get_user)(vcpu, r, &val);
3694 } else {
3695 val = __vcpu_sys_reg(vcpu, r->reg);
3696 ret = 0;
3697 }
3698
3699 if (!ret)
3700 ret = put_user(val, uaddr);
3701
3702 return ret;
3703 }
3704
kvm_arm_sys_reg_get_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)3705 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3706 {
3707 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
3708 int err;
3709
3710 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
3711 return demux_c15_get(vcpu, reg->id, uaddr);
3712
3713 err = get_invariant_sys_reg(reg->id, uaddr);
3714 if (err != -ENOENT)
3715 return err;
3716
3717 return kvm_sys_reg_get_user(vcpu, reg,
3718 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3719 }
3720
kvm_sys_reg_set_user(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg,const struct sys_reg_desc table[],unsigned int num)3721 int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3722 const struct sys_reg_desc table[], unsigned int num)
3723 {
3724 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
3725 const struct sys_reg_desc *r;
3726 u64 val;
3727 int ret;
3728
3729 if (get_user(val, uaddr))
3730 return -EFAULT;
3731
3732 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
3733 if (!r || sysreg_hidden_user(vcpu, r))
3734 return -ENOENT;
3735
3736 if (sysreg_user_write_ignore(vcpu, r))
3737 return 0;
3738
3739 if (r->set_user) {
3740 ret = (r->set_user)(vcpu, r, val);
3741 } else {
3742 __vcpu_sys_reg(vcpu, r->reg) = val;
3743 ret = 0;
3744 }
3745
3746 return ret;
3747 }
3748
kvm_arm_sys_reg_set_reg(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)3749 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3750 {
3751 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
3752 int err;
3753
3754 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
3755 return demux_c15_set(vcpu, reg->id, uaddr);
3756
3757 err = set_invariant_sys_reg(reg->id, uaddr);
3758 if (err != -ENOENT)
3759 return err;
3760
3761 return kvm_sys_reg_set_user(vcpu, reg,
3762 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3763 }
3764
num_demux_regs(void)3765 static unsigned int num_demux_regs(void)
3766 {
3767 return CSSELR_MAX;
3768 }
3769
write_demux_regids(u64 __user * uindices)3770 static int write_demux_regids(u64 __user *uindices)
3771 {
3772 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
3773 unsigned int i;
3774
3775 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
3776 for (i = 0; i < CSSELR_MAX; i++) {
3777 if (put_user(val | i, uindices))
3778 return -EFAULT;
3779 uindices++;
3780 }
3781 return 0;
3782 }
3783
sys_reg_to_index(const struct sys_reg_desc * reg)3784 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
3785 {
3786 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
3787 KVM_REG_ARM64_SYSREG |
3788 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
3789 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
3790 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
3791 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
3792 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
3793 }
3794
copy_reg_to_user(const struct sys_reg_desc * reg,u64 __user ** uind)3795 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
3796 {
3797 if (!*uind)
3798 return true;
3799
3800 if (put_user(sys_reg_to_index(reg), *uind))
3801 return false;
3802
3803 (*uind)++;
3804 return true;
3805 }
3806
walk_one_sys_reg(const struct kvm_vcpu * vcpu,const struct sys_reg_desc * rd,u64 __user ** uind,unsigned int * total)3807 static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
3808 const struct sys_reg_desc *rd,
3809 u64 __user **uind,
3810 unsigned int *total)
3811 {
3812 /*
3813 * Ignore registers we trap but don't save,
3814 * and for which no custom user accessor is provided.
3815 */
3816 if (!(rd->reg || rd->get_user))
3817 return 0;
3818
3819 if (sysreg_hidden_user(vcpu, rd))
3820 return 0;
3821
3822 if (!copy_reg_to_user(rd, uind))
3823 return -EFAULT;
3824
3825 (*total)++;
3826 return 0;
3827 }
3828
3829 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
walk_sys_regs(struct kvm_vcpu * vcpu,u64 __user * uind)3830 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
3831 {
3832 const struct sys_reg_desc *i2, *end2;
3833 unsigned int total = 0;
3834 int err;
3835
3836 i2 = sys_reg_descs;
3837 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
3838
3839 while (i2 != end2) {
3840 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
3841 if (err)
3842 return err;
3843 }
3844 return total;
3845 }
3846
kvm_arm_num_sys_reg_descs(struct kvm_vcpu * vcpu)3847 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
3848 {
3849 return ARRAY_SIZE(invariant_sys_regs)
3850 + num_demux_regs()
3851 + walk_sys_regs(vcpu, (u64 __user *)NULL);
3852 }
3853
kvm_arm_copy_sys_reg_indices(struct kvm_vcpu * vcpu,u64 __user * uindices)3854 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
3855 {
3856 unsigned int i;
3857 int err;
3858
3859 /* Then give them all the invariant registers' indices. */
3860 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
3861 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
3862 return -EFAULT;
3863 uindices++;
3864 }
3865
3866 err = walk_sys_regs(vcpu, uindices);
3867 if (err < 0)
3868 return err;
3869 uindices += err;
3870
3871 return write_demux_regids(uindices);
3872 }
3873
3874 #define KVM_ARM_FEATURE_ID_RANGE_INDEX(r) \
3875 KVM_ARM_FEATURE_ID_RANGE_IDX(sys_reg_Op0(r), \
3876 sys_reg_Op1(r), \
3877 sys_reg_CRn(r), \
3878 sys_reg_CRm(r), \
3879 sys_reg_Op2(r))
3880
is_feature_id_reg(u32 encoding)3881 static bool is_feature_id_reg(u32 encoding)
3882 {
3883 return (sys_reg_Op0(encoding) == 3 &&
3884 (sys_reg_Op1(encoding) < 2 || sys_reg_Op1(encoding) == 3) &&
3885 sys_reg_CRn(encoding) == 0 &&
3886 sys_reg_CRm(encoding) <= 7);
3887 }
3888
kvm_vm_ioctl_get_reg_writable_masks(struct kvm * kvm,struct reg_mask_range * range)3889 int kvm_vm_ioctl_get_reg_writable_masks(struct kvm *kvm, struct reg_mask_range *range)
3890 {
3891 const void *zero_page = page_to_virt(ZERO_PAGE(0));
3892 u64 __user *masks = (u64 __user *)range->addr;
3893
3894 /* Only feature id range is supported, reserved[13] must be zero. */
3895 if (range->range ||
3896 memcmp(range->reserved, zero_page, sizeof(range->reserved)))
3897 return -EINVAL;
3898
3899 /* Wipe the whole thing first */
3900 if (clear_user(masks, KVM_ARM_FEATURE_ID_RANGE_SIZE * sizeof(__u64)))
3901 return -EFAULT;
3902
3903 for (int i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3904 const struct sys_reg_desc *reg = &sys_reg_descs[i];
3905 u32 encoding = reg_to_encoding(reg);
3906 u64 val;
3907
3908 if (!is_feature_id_reg(encoding) || !reg->set_user)
3909 continue;
3910
3911 /*
3912 * For ID registers, we return the writable mask. Other feature
3913 * registers return a full 64bit mask. That's not necessary
3914 * compliant with a given revision of the architecture, but the
3915 * RES0/RES1 definitions allow us to do that.
3916 */
3917 if (is_id_reg(encoding)) {
3918 if (!reg->val ||
3919 (is_aa32_id_reg(encoding) && !kvm_supports_32bit_el0()))
3920 continue;
3921 val = reg->val;
3922 } else {
3923 val = ~0UL;
3924 }
3925
3926 if (put_user(val, (masks + KVM_ARM_FEATURE_ID_RANGE_INDEX(encoding))))
3927 return -EFAULT;
3928 }
3929
3930 return 0;
3931 }
3932
kvm_sys_reg_table_init(void)3933 int __init kvm_sys_reg_table_init(void)
3934 {
3935 struct sys_reg_params params;
3936 bool valid = true;
3937 unsigned int i;
3938
3939 /* Make sure tables are unique and in order. */
3940 valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
3941 valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
3942 valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
3943 valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
3944 valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
3945 valid &= check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false);
3946
3947 if (!valid)
3948 return -EINVAL;
3949
3950 /* We abuse the reset function to overwrite the table itself. */
3951 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
3952 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
3953
3954 /* Find the first idreg (SYS_ID_PFR0_EL1) in sys_reg_descs. */
3955 params = encoding_to_params(SYS_ID_PFR0_EL1);
3956 first_idreg = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3957 if (!first_idreg)
3958 return -EINVAL;
3959
3960 if (kvm_get_mode() == KVM_MODE_NV)
3961 return populate_nv_trap_config();
3962
3963 return 0;
3964 }
3965