1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Check for KVM_GET_REG_LIST regressions.
4  *
5  * Copyright (C) 2020, Red Hat, Inc.
6  *
7  * While the blessed list should be created from the oldest possible
8  * kernel, we can't go older than v5.2, though, because that's the first
9  * release which includes df205b5c6328 ("KVM: arm64: Filter out invalid
10  * core register IDs in KVM_GET_REG_LIST"). Without that commit the core
11  * registers won't match expectations.
12  */
13 #include <stdio.h>
14 #include "kvm_util.h"
15 #include "test_util.h"
16 #include "processor.h"
17 
18 struct feature_id_reg {
19 	__u64 reg;
20 	__u64 id_reg;
21 	__u64 feat_shift;
22 	__u64 feat_min;
23 };
24 
25 static struct feature_id_reg feat_id_regs[] = {
26 	{
27 		ARM64_SYS_REG(3, 0, 2, 0, 3),	/* TCR2_EL1 */
28 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
29 		0,
30 		1
31 	},
32 	{
33 		ARM64_SYS_REG(3, 0, 10, 2, 2),	/* PIRE0_EL1 */
34 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
35 		8,
36 		1
37 	},
38 	{
39 		ARM64_SYS_REG(3, 0, 10, 2, 3),	/* PIR_EL1 */
40 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
41 		8,
42 		1
43 	},
44 	{
45 		ARM64_SYS_REG(3, 0, 10, 2, 4),	/* POR_EL1 */
46 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
47 		16,
48 		1
49 	},
50 	{
51 		ARM64_SYS_REG(3, 3, 10, 2, 4),	/* POR_EL0 */
52 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
53 		16,
54 		1
55 	}
56 };
57 
filter_reg(__u64 reg)58 bool filter_reg(__u64 reg)
59 {
60 	/*
61 	 * DEMUX register presence depends on the host's CLIDR_EL1.
62 	 * This means there's no set of them that we can bless.
63 	 */
64 	if ((reg & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
65 		return true;
66 
67 	return false;
68 }
69 
check_supported_feat_reg(struct kvm_vcpu * vcpu,__u64 reg)70 static bool check_supported_feat_reg(struct kvm_vcpu *vcpu, __u64 reg)
71 {
72 	int i, ret;
73 	__u64 data, feat_val;
74 
75 	for (i = 0; i < ARRAY_SIZE(feat_id_regs); i++) {
76 		if (feat_id_regs[i].reg == reg) {
77 			ret = __vcpu_get_reg(vcpu, feat_id_regs[i].id_reg, &data);
78 			if (ret < 0)
79 				return false;
80 
81 			feat_val = ((data >> feat_id_regs[i].feat_shift) & 0xf);
82 			return feat_val >= feat_id_regs[i].feat_min;
83 		}
84 	}
85 
86 	return true;
87 }
88 
check_supported_reg(struct kvm_vcpu * vcpu,__u64 reg)89 bool check_supported_reg(struct kvm_vcpu *vcpu, __u64 reg)
90 {
91 	return check_supported_feat_reg(vcpu, reg);
92 }
93 
check_reject_set(int err)94 bool check_reject_set(int err)
95 {
96 	return err == EPERM;
97 }
98 
finalize_vcpu(struct kvm_vcpu * vcpu,struct vcpu_reg_list * c)99 void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c)
100 {
101 	struct vcpu_reg_sublist *s;
102 	int feature;
103 
104 	for_each_sublist(c, s) {
105 		if (s->finalize) {
106 			feature = s->feature;
107 			vcpu_ioctl(vcpu, KVM_ARM_VCPU_FINALIZE, &feature);
108 		}
109 	}
110 }
111 
112 #define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_COPROC_MASK)
113 
114 #define CORE_REGS_XX_NR_WORDS	2
115 #define CORE_SPSR_XX_NR_WORDS	2
116 #define CORE_FPREGS_XX_NR_WORDS	4
117 
core_id_to_str(const char * prefix,__u64 id)118 static const char *core_id_to_str(const char *prefix, __u64 id)
119 {
120 	__u64 core_off = id & ~REG_MASK, idx;
121 
122 	/*
123 	 * core_off is the offset into struct kvm_regs
124 	 */
125 	switch (core_off) {
126 	case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
127 	     KVM_REG_ARM_CORE_REG(regs.regs[30]):
128 		idx = (core_off - KVM_REG_ARM_CORE_REG(regs.regs[0])) / CORE_REGS_XX_NR_WORDS;
129 		TEST_ASSERT(idx < 31, "%s: Unexpected regs.regs index: %lld", prefix, idx);
130 		return strdup_printf("KVM_REG_ARM_CORE_REG(regs.regs[%lld])", idx);
131 	case KVM_REG_ARM_CORE_REG(regs.sp):
132 		return "KVM_REG_ARM_CORE_REG(regs.sp)";
133 	case KVM_REG_ARM_CORE_REG(regs.pc):
134 		return "KVM_REG_ARM_CORE_REG(regs.pc)";
135 	case KVM_REG_ARM_CORE_REG(regs.pstate):
136 		return "KVM_REG_ARM_CORE_REG(regs.pstate)";
137 	case KVM_REG_ARM_CORE_REG(sp_el1):
138 		return "KVM_REG_ARM_CORE_REG(sp_el1)";
139 	case KVM_REG_ARM_CORE_REG(elr_el1):
140 		return "KVM_REG_ARM_CORE_REG(elr_el1)";
141 	case KVM_REG_ARM_CORE_REG(spsr[0]) ...
142 	     KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
143 		idx = (core_off - KVM_REG_ARM_CORE_REG(spsr[0])) / CORE_SPSR_XX_NR_WORDS;
144 		TEST_ASSERT(idx < KVM_NR_SPSR, "%s: Unexpected spsr index: %lld", prefix, idx);
145 		return strdup_printf("KVM_REG_ARM_CORE_REG(spsr[%lld])", idx);
146 	case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
147 	     KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
148 		idx = (core_off - KVM_REG_ARM_CORE_REG(fp_regs.vregs[0])) / CORE_FPREGS_XX_NR_WORDS;
149 		TEST_ASSERT(idx < 32, "%s: Unexpected fp_regs.vregs index: %lld", prefix, idx);
150 		return strdup_printf("KVM_REG_ARM_CORE_REG(fp_regs.vregs[%lld])", idx);
151 	case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
152 		return "KVM_REG_ARM_CORE_REG(fp_regs.fpsr)";
153 	case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
154 		return "KVM_REG_ARM_CORE_REG(fp_regs.fpcr)";
155 	}
156 
157 	TEST_FAIL("%s: Unknown core reg id: 0x%llx", prefix, id);
158 	return NULL;
159 }
160 
sve_id_to_str(const char * prefix,__u64 id)161 static const char *sve_id_to_str(const char *prefix, __u64 id)
162 {
163 	__u64 sve_off, n, i;
164 
165 	if (id == KVM_REG_ARM64_SVE_VLS)
166 		return "KVM_REG_ARM64_SVE_VLS";
167 
168 	sve_off = id & ~(REG_MASK | ((1ULL << 5) - 1));
169 	i = id & (KVM_ARM64_SVE_MAX_SLICES - 1);
170 
171 	TEST_ASSERT(i == 0, "%s: Currently we don't expect slice > 0, reg id 0x%llx", prefix, id);
172 
173 	switch (sve_off) {
174 	case KVM_REG_ARM64_SVE_ZREG_BASE ...
175 	     KVM_REG_ARM64_SVE_ZREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_ZREGS - 1:
176 		n = (id >> 5) & (KVM_ARM64_SVE_NUM_ZREGS - 1);
177 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_ZREG(n, 0),
178 			    "%s: Unexpected bits set in SVE ZREG id: 0x%llx", prefix, id);
179 		return strdup_printf("KVM_REG_ARM64_SVE_ZREG(%lld, 0)", n);
180 	case KVM_REG_ARM64_SVE_PREG_BASE ...
181 	     KVM_REG_ARM64_SVE_PREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_PREGS - 1:
182 		n = (id >> 5) & (KVM_ARM64_SVE_NUM_PREGS - 1);
183 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_PREG(n, 0),
184 			    "%s: Unexpected bits set in SVE PREG id: 0x%llx", prefix, id);
185 		return strdup_printf("KVM_REG_ARM64_SVE_PREG(%lld, 0)", n);
186 	case KVM_REG_ARM64_SVE_FFR_BASE:
187 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_FFR(0),
188 			    "%s: Unexpected bits set in SVE FFR id: 0x%llx", prefix, id);
189 		return "KVM_REG_ARM64_SVE_FFR(0)";
190 	}
191 
192 	return NULL;
193 }
194 
print_reg(const char * prefix,__u64 id)195 void print_reg(const char *prefix, __u64 id)
196 {
197 	unsigned op0, op1, crn, crm, op2;
198 	const char *reg_size = NULL;
199 
200 	TEST_ASSERT((id & KVM_REG_ARCH_MASK) == KVM_REG_ARM64,
201 		    "%s: KVM_REG_ARM64 missing in reg id: 0x%llx", prefix, id);
202 
203 	switch (id & KVM_REG_SIZE_MASK) {
204 	case KVM_REG_SIZE_U8:
205 		reg_size = "KVM_REG_SIZE_U8";
206 		break;
207 	case KVM_REG_SIZE_U16:
208 		reg_size = "KVM_REG_SIZE_U16";
209 		break;
210 	case KVM_REG_SIZE_U32:
211 		reg_size = "KVM_REG_SIZE_U32";
212 		break;
213 	case KVM_REG_SIZE_U64:
214 		reg_size = "KVM_REG_SIZE_U64";
215 		break;
216 	case KVM_REG_SIZE_U128:
217 		reg_size = "KVM_REG_SIZE_U128";
218 		break;
219 	case KVM_REG_SIZE_U256:
220 		reg_size = "KVM_REG_SIZE_U256";
221 		break;
222 	case KVM_REG_SIZE_U512:
223 		reg_size = "KVM_REG_SIZE_U512";
224 		break;
225 	case KVM_REG_SIZE_U1024:
226 		reg_size = "KVM_REG_SIZE_U1024";
227 		break;
228 	case KVM_REG_SIZE_U2048:
229 		reg_size = "KVM_REG_SIZE_U2048";
230 		break;
231 	default:
232 		TEST_FAIL("%s: Unexpected reg size: 0x%llx in reg id: 0x%llx",
233 			  prefix, (id & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT, id);
234 	}
235 
236 	switch (id & KVM_REG_ARM_COPROC_MASK) {
237 	case KVM_REG_ARM_CORE:
238 		printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_CORE | %s,\n", reg_size, core_id_to_str(prefix, id));
239 		break;
240 	case KVM_REG_ARM_DEMUX:
241 		TEST_ASSERT(!(id & ~(REG_MASK | KVM_REG_ARM_DEMUX_ID_MASK | KVM_REG_ARM_DEMUX_VAL_MASK)),
242 			    "%s: Unexpected bits set in DEMUX reg id: 0x%llx", prefix, id);
243 		printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_DEMUX | KVM_REG_ARM_DEMUX_ID_CCSIDR | %lld,\n",
244 		       reg_size, id & KVM_REG_ARM_DEMUX_VAL_MASK);
245 		break;
246 	case KVM_REG_ARM64_SYSREG:
247 		op0 = (id & KVM_REG_ARM64_SYSREG_OP0_MASK) >> KVM_REG_ARM64_SYSREG_OP0_SHIFT;
248 		op1 = (id & KVM_REG_ARM64_SYSREG_OP1_MASK) >> KVM_REG_ARM64_SYSREG_OP1_SHIFT;
249 		crn = (id & KVM_REG_ARM64_SYSREG_CRN_MASK) >> KVM_REG_ARM64_SYSREG_CRN_SHIFT;
250 		crm = (id & KVM_REG_ARM64_SYSREG_CRM_MASK) >> KVM_REG_ARM64_SYSREG_CRM_SHIFT;
251 		op2 = (id & KVM_REG_ARM64_SYSREG_OP2_MASK) >> KVM_REG_ARM64_SYSREG_OP2_SHIFT;
252 		TEST_ASSERT(id == ARM64_SYS_REG(op0, op1, crn, crm, op2),
253 			    "%s: Unexpected bits set in SYSREG reg id: 0x%llx", prefix, id);
254 		printf("\tARM64_SYS_REG(%d, %d, %d, %d, %d),\n", op0, op1, crn, crm, op2);
255 		break;
256 	case KVM_REG_ARM_FW:
257 		TEST_ASSERT(id == KVM_REG_ARM_FW_REG(id & 0xffff),
258 			    "%s: Unexpected bits set in FW reg id: 0x%llx", prefix, id);
259 		printf("\tKVM_REG_ARM_FW_REG(%lld),\n", id & 0xffff);
260 		break;
261 	case KVM_REG_ARM_FW_FEAT_BMAP:
262 		TEST_ASSERT(id == KVM_REG_ARM_FW_FEAT_BMAP_REG(id & 0xffff),
263 			    "%s: Unexpected bits set in the bitmap feature FW reg id: 0x%llx", prefix, id);
264 		printf("\tKVM_REG_ARM_FW_FEAT_BMAP_REG(%lld),\n", id & 0xffff);
265 		break;
266 	case KVM_REG_ARM64_SVE:
267 		printf("\t%s,\n", sve_id_to_str(prefix, id));
268 		break;
269 	default:
270 		TEST_FAIL("%s: Unexpected coproc type: 0x%llx in reg id: 0x%llx",
271 			  prefix, (id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT, id);
272 	}
273 }
274 
275 /*
276  * The original blessed list was primed with the output of kernel version
277  * v4.15 with --core-reg-fixup and then later updated with new registers.
278  * (The --core-reg-fixup option and it's fixup function have been removed
279  * from the test, as it's unlikely to use this type of test on a kernel
280  * older than v5.2.)
281  *
282  * The blessed list is up to date with kernel version v6.4 (or so we hope)
283  */
284 static __u64 base_regs[] = {
285 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[0]),
286 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[1]),
287 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[2]),
288 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[3]),
289 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[4]),
290 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[5]),
291 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[6]),
292 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[7]),
293 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[8]),
294 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[9]),
295 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[10]),
296 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[11]),
297 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[12]),
298 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[13]),
299 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[14]),
300 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[15]),
301 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[16]),
302 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[17]),
303 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[18]),
304 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[19]),
305 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[20]),
306 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[21]),
307 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[22]),
308 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[23]),
309 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[24]),
310 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[25]),
311 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[26]),
312 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[27]),
313 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[28]),
314 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[29]),
315 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[30]),
316 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.sp),
317 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.pc),
318 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.pstate),
319 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(sp_el1),
320 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(elr_el1),
321 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[0]),
322 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[1]),
323 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[2]),
324 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[3]),
325 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[4]),
326 	KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.fpsr),
327 	KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.fpcr),
328 	KVM_REG_ARM_FW_REG(0),		/* KVM_REG_ARM_PSCI_VERSION */
329 	KVM_REG_ARM_FW_REG(1),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1 */
330 	KVM_REG_ARM_FW_REG(2),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2 */
331 	KVM_REG_ARM_FW_REG(3),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 */
332 	KVM_REG_ARM_FW_FEAT_BMAP_REG(0),	/* KVM_REG_ARM_STD_BMAP */
333 	KVM_REG_ARM_FW_FEAT_BMAP_REG(1),	/* KVM_REG_ARM_STD_HYP_BMAP */
334 	KVM_REG_ARM_FW_FEAT_BMAP_REG(2),	/* KVM_REG_ARM_VENDOR_HYP_BMAP */
335 	KVM_REG_ARM_FW_FEAT_BMAP_REG(3),	/* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
336 	ARM64_SYS_REG(3, 3, 14, 3, 1),	/* CNTV_CTL_EL0 */
337 	ARM64_SYS_REG(3, 3, 14, 3, 2),	/* CNTV_CVAL_EL0 */
338 	ARM64_SYS_REG(3, 3, 14, 0, 2),
339 	ARM64_SYS_REG(3, 0, 0, 0, 0),	/* MIDR_EL1 */
340 	ARM64_SYS_REG(3, 0, 0, 0, 6),	/* REVIDR_EL1 */
341 	ARM64_SYS_REG(3, 1, 0, 0, 1),	/* CLIDR_EL1 */
342 	ARM64_SYS_REG(3, 1, 0, 0, 7),	/* AIDR_EL1 */
343 	ARM64_SYS_REG(3, 3, 0, 0, 1),	/* CTR_EL0 */
344 	ARM64_SYS_REG(2, 0, 0, 0, 4),
345 	ARM64_SYS_REG(2, 0, 0, 0, 5),
346 	ARM64_SYS_REG(2, 0, 0, 0, 6),
347 	ARM64_SYS_REG(2, 0, 0, 0, 7),
348 	ARM64_SYS_REG(2, 0, 0, 1, 4),
349 	ARM64_SYS_REG(2, 0, 0, 1, 5),
350 	ARM64_SYS_REG(2, 0, 0, 1, 6),
351 	ARM64_SYS_REG(2, 0, 0, 1, 7),
352 	ARM64_SYS_REG(2, 0, 0, 2, 0),	/* MDCCINT_EL1 */
353 	ARM64_SYS_REG(2, 0, 0, 2, 2),	/* MDSCR_EL1 */
354 	ARM64_SYS_REG(2, 0, 0, 2, 4),
355 	ARM64_SYS_REG(2, 0, 0, 2, 5),
356 	ARM64_SYS_REG(2, 0, 0, 2, 6),
357 	ARM64_SYS_REG(2, 0, 0, 2, 7),
358 	ARM64_SYS_REG(2, 0, 0, 3, 4),
359 	ARM64_SYS_REG(2, 0, 0, 3, 5),
360 	ARM64_SYS_REG(2, 0, 0, 3, 6),
361 	ARM64_SYS_REG(2, 0, 0, 3, 7),
362 	ARM64_SYS_REG(2, 0, 0, 4, 4),
363 	ARM64_SYS_REG(2, 0, 0, 4, 5),
364 	ARM64_SYS_REG(2, 0, 0, 4, 6),
365 	ARM64_SYS_REG(2, 0, 0, 4, 7),
366 	ARM64_SYS_REG(2, 0, 0, 5, 4),
367 	ARM64_SYS_REG(2, 0, 0, 5, 5),
368 	ARM64_SYS_REG(2, 0, 0, 5, 6),
369 	ARM64_SYS_REG(2, 0, 0, 5, 7),
370 	ARM64_SYS_REG(2, 0, 0, 6, 4),
371 	ARM64_SYS_REG(2, 0, 0, 6, 5),
372 	ARM64_SYS_REG(2, 0, 0, 6, 6),
373 	ARM64_SYS_REG(2, 0, 0, 6, 7),
374 	ARM64_SYS_REG(2, 0, 0, 7, 4),
375 	ARM64_SYS_REG(2, 0, 0, 7, 5),
376 	ARM64_SYS_REG(2, 0, 0, 7, 6),
377 	ARM64_SYS_REG(2, 0, 0, 7, 7),
378 	ARM64_SYS_REG(2, 0, 0, 8, 4),
379 	ARM64_SYS_REG(2, 0, 0, 8, 5),
380 	ARM64_SYS_REG(2, 0, 0, 8, 6),
381 	ARM64_SYS_REG(2, 0, 0, 8, 7),
382 	ARM64_SYS_REG(2, 0, 0, 9, 4),
383 	ARM64_SYS_REG(2, 0, 0, 9, 5),
384 	ARM64_SYS_REG(2, 0, 0, 9, 6),
385 	ARM64_SYS_REG(2, 0, 0, 9, 7),
386 	ARM64_SYS_REG(2, 0, 0, 10, 4),
387 	ARM64_SYS_REG(2, 0, 0, 10, 5),
388 	ARM64_SYS_REG(2, 0, 0, 10, 6),
389 	ARM64_SYS_REG(2, 0, 0, 10, 7),
390 	ARM64_SYS_REG(2, 0, 0, 11, 4),
391 	ARM64_SYS_REG(2, 0, 0, 11, 5),
392 	ARM64_SYS_REG(2, 0, 0, 11, 6),
393 	ARM64_SYS_REG(2, 0, 0, 11, 7),
394 	ARM64_SYS_REG(2, 0, 0, 12, 4),
395 	ARM64_SYS_REG(2, 0, 0, 12, 5),
396 	ARM64_SYS_REG(2, 0, 0, 12, 6),
397 	ARM64_SYS_REG(2, 0, 0, 12, 7),
398 	ARM64_SYS_REG(2, 0, 0, 13, 4),
399 	ARM64_SYS_REG(2, 0, 0, 13, 5),
400 	ARM64_SYS_REG(2, 0, 0, 13, 6),
401 	ARM64_SYS_REG(2, 0, 0, 13, 7),
402 	ARM64_SYS_REG(2, 0, 0, 14, 4),
403 	ARM64_SYS_REG(2, 0, 0, 14, 5),
404 	ARM64_SYS_REG(2, 0, 0, 14, 6),
405 	ARM64_SYS_REG(2, 0, 0, 14, 7),
406 	ARM64_SYS_REG(2, 0, 0, 15, 4),
407 	ARM64_SYS_REG(2, 0, 0, 15, 5),
408 	ARM64_SYS_REG(2, 0, 0, 15, 6),
409 	ARM64_SYS_REG(2, 0, 0, 15, 7),
410 	ARM64_SYS_REG(2, 0, 1, 1, 4),	/* OSLSR_EL1 */
411 	ARM64_SYS_REG(2, 4, 0, 7, 0),	/* DBGVCR32_EL2 */
412 	ARM64_SYS_REG(3, 0, 0, 0, 5),	/* MPIDR_EL1 */
413 	ARM64_SYS_REG(3, 0, 0, 1, 0),	/* ID_PFR0_EL1 */
414 	ARM64_SYS_REG(3, 0, 0, 1, 1),	/* ID_PFR1_EL1 */
415 	ARM64_SYS_REG(3, 0, 0, 1, 2),	/* ID_DFR0_EL1 */
416 	ARM64_SYS_REG(3, 0, 0, 1, 3),	/* ID_AFR0_EL1 */
417 	ARM64_SYS_REG(3, 0, 0, 1, 4),	/* ID_MMFR0_EL1 */
418 	ARM64_SYS_REG(3, 0, 0, 1, 5),	/* ID_MMFR1_EL1 */
419 	ARM64_SYS_REG(3, 0, 0, 1, 6),	/* ID_MMFR2_EL1 */
420 	ARM64_SYS_REG(3, 0, 0, 1, 7),	/* ID_MMFR3_EL1 */
421 	ARM64_SYS_REG(3, 0, 0, 2, 0),	/* ID_ISAR0_EL1 */
422 	ARM64_SYS_REG(3, 0, 0, 2, 1),	/* ID_ISAR1_EL1 */
423 	ARM64_SYS_REG(3, 0, 0, 2, 2),	/* ID_ISAR2_EL1 */
424 	ARM64_SYS_REG(3, 0, 0, 2, 3),	/* ID_ISAR3_EL1 */
425 	ARM64_SYS_REG(3, 0, 0, 2, 4),	/* ID_ISAR4_EL1 */
426 	ARM64_SYS_REG(3, 0, 0, 2, 5),	/* ID_ISAR5_EL1 */
427 	ARM64_SYS_REG(3, 0, 0, 2, 6),	/* ID_MMFR4_EL1 */
428 	ARM64_SYS_REG(3, 0, 0, 2, 7),	/* ID_ISAR6_EL1 */
429 	ARM64_SYS_REG(3, 0, 0, 3, 0),	/* MVFR0_EL1 */
430 	ARM64_SYS_REG(3, 0, 0, 3, 1),	/* MVFR1_EL1 */
431 	ARM64_SYS_REG(3, 0, 0, 3, 2),	/* MVFR2_EL1 */
432 	ARM64_SYS_REG(3, 0, 0, 3, 3),
433 	ARM64_SYS_REG(3, 0, 0, 3, 4),	/* ID_PFR2_EL1 */
434 	ARM64_SYS_REG(3, 0, 0, 3, 5),	/* ID_DFR1_EL1 */
435 	ARM64_SYS_REG(3, 0, 0, 3, 6),	/* ID_MMFR5_EL1 */
436 	ARM64_SYS_REG(3, 0, 0, 3, 7),
437 	ARM64_SYS_REG(3, 0, 0, 4, 0),	/* ID_AA64PFR0_EL1 */
438 	ARM64_SYS_REG(3, 0, 0, 4, 1),	/* ID_AA64PFR1_EL1 */
439 	ARM64_SYS_REG(3, 0, 0, 4, 2),	/* ID_AA64PFR2_EL1 */
440 	ARM64_SYS_REG(3, 0, 0, 4, 3),
441 	ARM64_SYS_REG(3, 0, 0, 4, 4),	/* ID_AA64ZFR0_EL1 */
442 	ARM64_SYS_REG(3, 0, 0, 4, 5),	/* ID_AA64SMFR0_EL1 */
443 	ARM64_SYS_REG(3, 0, 0, 4, 6),
444 	ARM64_SYS_REG(3, 0, 0, 4, 7),
445 	ARM64_SYS_REG(3, 0, 0, 5, 0),	/* ID_AA64DFR0_EL1 */
446 	ARM64_SYS_REG(3, 0, 0, 5, 1),	/* ID_AA64DFR1_EL1 */
447 	ARM64_SYS_REG(3, 0, 0, 5, 2),
448 	ARM64_SYS_REG(3, 0, 0, 5, 3),
449 	ARM64_SYS_REG(3, 0, 0, 5, 4),	/* ID_AA64AFR0_EL1 */
450 	ARM64_SYS_REG(3, 0, 0, 5, 5),	/* ID_AA64AFR1_EL1 */
451 	ARM64_SYS_REG(3, 0, 0, 5, 6),
452 	ARM64_SYS_REG(3, 0, 0, 5, 7),
453 	ARM64_SYS_REG(3, 0, 0, 6, 0),	/* ID_AA64ISAR0_EL1 */
454 	ARM64_SYS_REG(3, 0, 0, 6, 1),	/* ID_AA64ISAR1_EL1 */
455 	ARM64_SYS_REG(3, 0, 0, 6, 2),	/* ID_AA64ISAR2_EL1 */
456 	ARM64_SYS_REG(3, 0, 0, 6, 3),
457 	ARM64_SYS_REG(3, 0, 0, 6, 4),
458 	ARM64_SYS_REG(3, 0, 0, 6, 5),
459 	ARM64_SYS_REG(3, 0, 0, 6, 6),
460 	ARM64_SYS_REG(3, 0, 0, 6, 7),
461 	ARM64_SYS_REG(3, 0, 0, 7, 0),	/* ID_AA64MMFR0_EL1 */
462 	ARM64_SYS_REG(3, 0, 0, 7, 1),	/* ID_AA64MMFR1_EL1 */
463 	ARM64_SYS_REG(3, 0, 0, 7, 2),	/* ID_AA64MMFR2_EL1 */
464 	ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
465 	ARM64_SYS_REG(3, 0, 0, 7, 4),	/* ID_AA64MMFR4_EL1 */
466 	ARM64_SYS_REG(3, 0, 0, 7, 5),
467 	ARM64_SYS_REG(3, 0, 0, 7, 6),
468 	ARM64_SYS_REG(3, 0, 0, 7, 7),
469 	ARM64_SYS_REG(3, 0, 1, 0, 0),	/* SCTLR_EL1 */
470 	ARM64_SYS_REG(3, 0, 1, 0, 1),	/* ACTLR_EL1 */
471 	ARM64_SYS_REG(3, 0, 1, 0, 2),	/* CPACR_EL1 */
472 	ARM64_SYS_REG(3, 0, 2, 0, 0),	/* TTBR0_EL1 */
473 	ARM64_SYS_REG(3, 0, 2, 0, 1),	/* TTBR1_EL1 */
474 	ARM64_SYS_REG(3, 0, 2, 0, 2),	/* TCR_EL1 */
475 	ARM64_SYS_REG(3, 0, 2, 0, 3),	/* TCR2_EL1 */
476 	ARM64_SYS_REG(3, 0, 5, 1, 0),	/* AFSR0_EL1 */
477 	ARM64_SYS_REG(3, 0, 5, 1, 1),	/* AFSR1_EL1 */
478 	ARM64_SYS_REG(3, 0, 5, 2, 0),	/* ESR_EL1 */
479 	ARM64_SYS_REG(3, 0, 6, 0, 0),	/* FAR_EL1 */
480 	ARM64_SYS_REG(3, 0, 7, 4, 0),	/* PAR_EL1 */
481 	ARM64_SYS_REG(3, 0, 10, 2, 0),	/* MAIR_EL1 */
482 	ARM64_SYS_REG(3, 0, 10, 2, 2),	/* PIRE0_EL1 */
483 	ARM64_SYS_REG(3, 0, 10, 2, 3),	/* PIR_EL1 */
484 	ARM64_SYS_REG(3, 0, 10, 2, 4),	/* POR_EL1 */
485 	ARM64_SYS_REG(3, 0, 10, 3, 0),	/* AMAIR_EL1 */
486 	ARM64_SYS_REG(3, 0, 12, 0, 0),	/* VBAR_EL1 */
487 	ARM64_SYS_REG(3, 0, 12, 1, 1),	/* DISR_EL1 */
488 	ARM64_SYS_REG(3, 0, 13, 0, 1),	/* CONTEXTIDR_EL1 */
489 	ARM64_SYS_REG(3, 0, 13, 0, 4),	/* TPIDR_EL1 */
490 	ARM64_SYS_REG(3, 0, 14, 1, 0),	/* CNTKCTL_EL1 */
491 	ARM64_SYS_REG(3, 2, 0, 0, 0),	/* CSSELR_EL1 */
492 	ARM64_SYS_REG(3, 3, 10, 2, 4),	/* POR_EL0 */
493 	ARM64_SYS_REG(3, 3, 13, 0, 2),	/* TPIDR_EL0 */
494 	ARM64_SYS_REG(3, 3, 13, 0, 3),	/* TPIDRRO_EL0 */
495 	ARM64_SYS_REG(3, 3, 14, 0, 1),	/* CNTPCT_EL0 */
496 	ARM64_SYS_REG(3, 3, 14, 2, 1),	/* CNTP_CTL_EL0 */
497 	ARM64_SYS_REG(3, 3, 14, 2, 2),	/* CNTP_CVAL_EL0 */
498 	ARM64_SYS_REG(3, 4, 3, 0, 0),	/* DACR32_EL2 */
499 	ARM64_SYS_REG(3, 4, 5, 0, 1),	/* IFSR32_EL2 */
500 	ARM64_SYS_REG(3, 4, 5, 3, 0),	/* FPEXC32_EL2 */
501 };
502 
503 static __u64 pmu_regs[] = {
504 	ARM64_SYS_REG(3, 0, 9, 14, 1),	/* PMINTENSET_EL1 */
505 	ARM64_SYS_REG(3, 0, 9, 14, 2),	/* PMINTENCLR_EL1 */
506 	ARM64_SYS_REG(3, 3, 9, 12, 0),	/* PMCR_EL0 */
507 	ARM64_SYS_REG(3, 3, 9, 12, 1),	/* PMCNTENSET_EL0 */
508 	ARM64_SYS_REG(3, 3, 9, 12, 2),	/* PMCNTENCLR_EL0 */
509 	ARM64_SYS_REG(3, 3, 9, 12, 3),	/* PMOVSCLR_EL0 */
510 	ARM64_SYS_REG(3, 3, 9, 12, 4),	/* PMSWINC_EL0 */
511 	ARM64_SYS_REG(3, 3, 9, 12, 5),	/* PMSELR_EL0 */
512 	ARM64_SYS_REG(3, 3, 9, 13, 0),	/* PMCCNTR_EL0 */
513 	ARM64_SYS_REG(3, 3, 9, 14, 0),	/* PMUSERENR_EL0 */
514 	ARM64_SYS_REG(3, 3, 9, 14, 3),	/* PMOVSSET_EL0 */
515 	ARM64_SYS_REG(3, 3, 14, 8, 0),
516 	ARM64_SYS_REG(3, 3, 14, 8, 1),
517 	ARM64_SYS_REG(3, 3, 14, 8, 2),
518 	ARM64_SYS_REG(3, 3, 14, 8, 3),
519 	ARM64_SYS_REG(3, 3, 14, 8, 4),
520 	ARM64_SYS_REG(3, 3, 14, 8, 5),
521 	ARM64_SYS_REG(3, 3, 14, 8, 6),
522 	ARM64_SYS_REG(3, 3, 14, 8, 7),
523 	ARM64_SYS_REG(3, 3, 14, 9, 0),
524 	ARM64_SYS_REG(3, 3, 14, 9, 1),
525 	ARM64_SYS_REG(3, 3, 14, 9, 2),
526 	ARM64_SYS_REG(3, 3, 14, 9, 3),
527 	ARM64_SYS_REG(3, 3, 14, 9, 4),
528 	ARM64_SYS_REG(3, 3, 14, 9, 5),
529 	ARM64_SYS_REG(3, 3, 14, 9, 6),
530 	ARM64_SYS_REG(3, 3, 14, 9, 7),
531 	ARM64_SYS_REG(3, 3, 14, 10, 0),
532 	ARM64_SYS_REG(3, 3, 14, 10, 1),
533 	ARM64_SYS_REG(3, 3, 14, 10, 2),
534 	ARM64_SYS_REG(3, 3, 14, 10, 3),
535 	ARM64_SYS_REG(3, 3, 14, 10, 4),
536 	ARM64_SYS_REG(3, 3, 14, 10, 5),
537 	ARM64_SYS_REG(3, 3, 14, 10, 6),
538 	ARM64_SYS_REG(3, 3, 14, 10, 7),
539 	ARM64_SYS_REG(3, 3, 14, 11, 0),
540 	ARM64_SYS_REG(3, 3, 14, 11, 1),
541 	ARM64_SYS_REG(3, 3, 14, 11, 2),
542 	ARM64_SYS_REG(3, 3, 14, 11, 3),
543 	ARM64_SYS_REG(3, 3, 14, 11, 4),
544 	ARM64_SYS_REG(3, 3, 14, 11, 5),
545 	ARM64_SYS_REG(3, 3, 14, 11, 6),
546 	ARM64_SYS_REG(3, 3, 14, 12, 0),
547 	ARM64_SYS_REG(3, 3, 14, 12, 1),
548 	ARM64_SYS_REG(3, 3, 14, 12, 2),
549 	ARM64_SYS_REG(3, 3, 14, 12, 3),
550 	ARM64_SYS_REG(3, 3, 14, 12, 4),
551 	ARM64_SYS_REG(3, 3, 14, 12, 5),
552 	ARM64_SYS_REG(3, 3, 14, 12, 6),
553 	ARM64_SYS_REG(3, 3, 14, 12, 7),
554 	ARM64_SYS_REG(3, 3, 14, 13, 0),
555 	ARM64_SYS_REG(3, 3, 14, 13, 1),
556 	ARM64_SYS_REG(3, 3, 14, 13, 2),
557 	ARM64_SYS_REG(3, 3, 14, 13, 3),
558 	ARM64_SYS_REG(3, 3, 14, 13, 4),
559 	ARM64_SYS_REG(3, 3, 14, 13, 5),
560 	ARM64_SYS_REG(3, 3, 14, 13, 6),
561 	ARM64_SYS_REG(3, 3, 14, 13, 7),
562 	ARM64_SYS_REG(3, 3, 14, 14, 0),
563 	ARM64_SYS_REG(3, 3, 14, 14, 1),
564 	ARM64_SYS_REG(3, 3, 14, 14, 2),
565 	ARM64_SYS_REG(3, 3, 14, 14, 3),
566 	ARM64_SYS_REG(3, 3, 14, 14, 4),
567 	ARM64_SYS_REG(3, 3, 14, 14, 5),
568 	ARM64_SYS_REG(3, 3, 14, 14, 6),
569 	ARM64_SYS_REG(3, 3, 14, 14, 7),
570 	ARM64_SYS_REG(3, 3, 14, 15, 0),
571 	ARM64_SYS_REG(3, 3, 14, 15, 1),
572 	ARM64_SYS_REG(3, 3, 14, 15, 2),
573 	ARM64_SYS_REG(3, 3, 14, 15, 3),
574 	ARM64_SYS_REG(3, 3, 14, 15, 4),
575 	ARM64_SYS_REG(3, 3, 14, 15, 5),
576 	ARM64_SYS_REG(3, 3, 14, 15, 6),
577 	ARM64_SYS_REG(3, 3, 14, 15, 7),	/* PMCCFILTR_EL0 */
578 };
579 
580 static __u64 vregs[] = {
581 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]),
582 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[1]),
583 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[2]),
584 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[3]),
585 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[4]),
586 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[5]),
587 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[6]),
588 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[7]),
589 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[8]),
590 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[9]),
591 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[10]),
592 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[11]),
593 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[12]),
594 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[13]),
595 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[14]),
596 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[15]),
597 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[16]),
598 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[17]),
599 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[18]),
600 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[19]),
601 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[20]),
602 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[21]),
603 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[22]),
604 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[23]),
605 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[24]),
606 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[25]),
607 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[26]),
608 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[27]),
609 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[28]),
610 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[29]),
611 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[30]),
612 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]),
613 };
614 
615 static __u64 sve_regs[] = {
616 	KVM_REG_ARM64_SVE_VLS,
617 	KVM_REG_ARM64_SVE_ZREG(0, 0),
618 	KVM_REG_ARM64_SVE_ZREG(1, 0),
619 	KVM_REG_ARM64_SVE_ZREG(2, 0),
620 	KVM_REG_ARM64_SVE_ZREG(3, 0),
621 	KVM_REG_ARM64_SVE_ZREG(4, 0),
622 	KVM_REG_ARM64_SVE_ZREG(5, 0),
623 	KVM_REG_ARM64_SVE_ZREG(6, 0),
624 	KVM_REG_ARM64_SVE_ZREG(7, 0),
625 	KVM_REG_ARM64_SVE_ZREG(8, 0),
626 	KVM_REG_ARM64_SVE_ZREG(9, 0),
627 	KVM_REG_ARM64_SVE_ZREG(10, 0),
628 	KVM_REG_ARM64_SVE_ZREG(11, 0),
629 	KVM_REG_ARM64_SVE_ZREG(12, 0),
630 	KVM_REG_ARM64_SVE_ZREG(13, 0),
631 	KVM_REG_ARM64_SVE_ZREG(14, 0),
632 	KVM_REG_ARM64_SVE_ZREG(15, 0),
633 	KVM_REG_ARM64_SVE_ZREG(16, 0),
634 	KVM_REG_ARM64_SVE_ZREG(17, 0),
635 	KVM_REG_ARM64_SVE_ZREG(18, 0),
636 	KVM_REG_ARM64_SVE_ZREG(19, 0),
637 	KVM_REG_ARM64_SVE_ZREG(20, 0),
638 	KVM_REG_ARM64_SVE_ZREG(21, 0),
639 	KVM_REG_ARM64_SVE_ZREG(22, 0),
640 	KVM_REG_ARM64_SVE_ZREG(23, 0),
641 	KVM_REG_ARM64_SVE_ZREG(24, 0),
642 	KVM_REG_ARM64_SVE_ZREG(25, 0),
643 	KVM_REG_ARM64_SVE_ZREG(26, 0),
644 	KVM_REG_ARM64_SVE_ZREG(27, 0),
645 	KVM_REG_ARM64_SVE_ZREG(28, 0),
646 	KVM_REG_ARM64_SVE_ZREG(29, 0),
647 	KVM_REG_ARM64_SVE_ZREG(30, 0),
648 	KVM_REG_ARM64_SVE_ZREG(31, 0),
649 	KVM_REG_ARM64_SVE_PREG(0, 0),
650 	KVM_REG_ARM64_SVE_PREG(1, 0),
651 	KVM_REG_ARM64_SVE_PREG(2, 0),
652 	KVM_REG_ARM64_SVE_PREG(3, 0),
653 	KVM_REG_ARM64_SVE_PREG(4, 0),
654 	KVM_REG_ARM64_SVE_PREG(5, 0),
655 	KVM_REG_ARM64_SVE_PREG(6, 0),
656 	KVM_REG_ARM64_SVE_PREG(7, 0),
657 	KVM_REG_ARM64_SVE_PREG(8, 0),
658 	KVM_REG_ARM64_SVE_PREG(9, 0),
659 	KVM_REG_ARM64_SVE_PREG(10, 0),
660 	KVM_REG_ARM64_SVE_PREG(11, 0),
661 	KVM_REG_ARM64_SVE_PREG(12, 0),
662 	KVM_REG_ARM64_SVE_PREG(13, 0),
663 	KVM_REG_ARM64_SVE_PREG(14, 0),
664 	KVM_REG_ARM64_SVE_PREG(15, 0),
665 	KVM_REG_ARM64_SVE_FFR(0),
666 	ARM64_SYS_REG(3, 0, 1, 2, 0),   /* ZCR_EL1 */
667 };
668 
669 static __u64 sve_rejects_set[] = {
670 	KVM_REG_ARM64_SVE_VLS,
671 };
672 
673 static __u64 pauth_addr_regs[] = {
674 	ARM64_SYS_REG(3, 0, 2, 1, 0),	/* APIAKEYLO_EL1 */
675 	ARM64_SYS_REG(3, 0, 2, 1, 1),	/* APIAKEYHI_EL1 */
676 	ARM64_SYS_REG(3, 0, 2, 1, 2),	/* APIBKEYLO_EL1 */
677 	ARM64_SYS_REG(3, 0, 2, 1, 3),	/* APIBKEYHI_EL1 */
678 	ARM64_SYS_REG(3, 0, 2, 2, 0),	/* APDAKEYLO_EL1 */
679 	ARM64_SYS_REG(3, 0, 2, 2, 1),	/* APDAKEYHI_EL1 */
680 	ARM64_SYS_REG(3, 0, 2, 2, 2),	/* APDBKEYLO_EL1 */
681 	ARM64_SYS_REG(3, 0, 2, 2, 3)	/* APDBKEYHI_EL1 */
682 };
683 
684 static __u64 pauth_generic_regs[] = {
685 	ARM64_SYS_REG(3, 0, 2, 3, 0),	/* APGAKEYLO_EL1 */
686 	ARM64_SYS_REG(3, 0, 2, 3, 1),	/* APGAKEYHI_EL1 */
687 };
688 
689 #define BASE_SUBLIST \
690 	{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
691 #define VREGS_SUBLIST \
692 	{ "vregs", .regs = vregs, .regs_n = ARRAY_SIZE(vregs), }
693 #define PMU_SUBLIST \
694 	{ "pmu", .capability = KVM_CAP_ARM_PMU_V3, .feature = KVM_ARM_VCPU_PMU_V3, \
695 	  .regs = pmu_regs, .regs_n = ARRAY_SIZE(pmu_regs), }
696 #define SVE_SUBLIST \
697 	{ "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \
698 	  .regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \
699 	  .rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), }
700 #define PAUTH_SUBLIST							\
701 	{								\
702 		.name 		= "pauth_address",			\
703 		.capability	= KVM_CAP_ARM_PTRAUTH_ADDRESS,		\
704 		.feature	= KVM_ARM_VCPU_PTRAUTH_ADDRESS,		\
705 		.regs		= pauth_addr_regs,			\
706 		.regs_n		= ARRAY_SIZE(pauth_addr_regs),		\
707 	},								\
708 	{								\
709 		.name 		= "pauth_generic",			\
710 		.capability	= KVM_CAP_ARM_PTRAUTH_GENERIC,		\
711 		.feature	= KVM_ARM_VCPU_PTRAUTH_GENERIC,		\
712 		.regs		= pauth_generic_regs,			\
713 		.regs_n		= ARRAY_SIZE(pauth_generic_regs),	\
714 	}
715 
716 static struct vcpu_reg_list vregs_config = {
717 	.sublists = {
718 	BASE_SUBLIST,
719 	VREGS_SUBLIST,
720 	{0},
721 	},
722 };
723 static struct vcpu_reg_list vregs_pmu_config = {
724 	.sublists = {
725 	BASE_SUBLIST,
726 	VREGS_SUBLIST,
727 	PMU_SUBLIST,
728 	{0},
729 	},
730 };
731 static struct vcpu_reg_list sve_config = {
732 	.sublists = {
733 	BASE_SUBLIST,
734 	SVE_SUBLIST,
735 	{0},
736 	},
737 };
738 static struct vcpu_reg_list sve_pmu_config = {
739 	.sublists = {
740 	BASE_SUBLIST,
741 	SVE_SUBLIST,
742 	PMU_SUBLIST,
743 	{0},
744 	},
745 };
746 static struct vcpu_reg_list pauth_config = {
747 	.sublists = {
748 	BASE_SUBLIST,
749 	VREGS_SUBLIST,
750 	PAUTH_SUBLIST,
751 	{0},
752 	},
753 };
754 static struct vcpu_reg_list pauth_pmu_config = {
755 	.sublists = {
756 	BASE_SUBLIST,
757 	VREGS_SUBLIST,
758 	PAUTH_SUBLIST,
759 	PMU_SUBLIST,
760 	{0},
761 	},
762 };
763 
764 struct vcpu_reg_list *vcpu_configs[] = {
765 	&vregs_config,
766 	&vregs_pmu_config,
767 	&sve_config,
768 	&sve_pmu_config,
769 	&pauth_config,
770 	&pauth_pmu_config,
771 };
772 int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);
773