1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Fault injection for both 32 and 64bit guests. 4 * 5 * Copyright (C) 2012,2013 - ARM Ltd 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 * 8 * Based on arch/arm/kvm/emulate.c 9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 10 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 11 */ 12 13 #include <hyp/adjust_pc.h> 14 #include <linux/kvm_host.h> 15 #include <asm/kvm_emulate.h> 16 #include <asm/kvm_mmu.h> 17 #include <asm/kvm_nested.h> 18 19 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__) 20 #error Hypervisor code only! 21 #endif 22 23 static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) 24 { 25 u64 val; 26 27 if (unlikely(vcpu_has_nv(vcpu))) 28 return vcpu_read_sys_reg(vcpu, reg); 29 else if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) && 30 __vcpu_read_sys_reg_from_cpu(reg, &val)) 31 return val; 32 33 return __vcpu_sys_reg(vcpu, reg); 34 } 35 36 static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) 37 { 38 if (unlikely(vcpu_has_nv(vcpu))) 39 vcpu_write_sys_reg(vcpu, val, reg); 40 else if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU) || 41 !__vcpu_write_sys_reg_to_cpu(val, reg)) 42 __vcpu_assign_sys_reg(vcpu, reg, val); 43 } 44 45 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode, 46 u64 val) 47 { 48 if (unlikely(vcpu_has_nv(vcpu))) { 49 if (target_mode == PSR_MODE_EL1h) 50 vcpu_write_sys_reg(vcpu, val, SPSR_EL1); 51 else 52 vcpu_write_sys_reg(vcpu, val, SPSR_EL2); 53 } else if (has_vhe()) { 54 write_sysreg_el1(val, SYS_SPSR); 55 } else { 56 __vcpu_assign_sys_reg(vcpu, SPSR_EL1, val); 57 } 58 } 59 60 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val) 61 { 62 if (has_vhe()) 63 write_sysreg(val, spsr_abt); 64 else 65 vcpu->arch.ctxt.spsr_abt = val; 66 } 67 68 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val) 69 { 70 if (has_vhe()) 71 write_sysreg(val, spsr_und); 72 else 73 vcpu->arch.ctxt.spsr_und = val; 74 } 75 76 /* 77 * This performs the exception entry at a given EL (@target_mode), stashing PC 78 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE. 79 * The EL passed to this function *must* be a non-secure, privileged mode with 80 * bit 0 being set (PSTATE.SP == 1). 81 * 82 * When an exception is taken, most PSTATE fields are left unchanged in the 83 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all 84 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx 85 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0. 86 * 87 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429. 88 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426. 89 * 90 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from 91 * MSB to LSB. 92 */ 93 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, 94 enum exception_type type) 95 { 96 unsigned long sctlr, vbar, old, new, mode; 97 u64 exc_offset; 98 99 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); 100 101 if (mode == target_mode) 102 exc_offset = CURRENT_EL_SP_ELx_VECTOR; 103 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode) 104 exc_offset = CURRENT_EL_SP_EL0_VECTOR; 105 else if (!(mode & PSR_MODE32_BIT)) 106 exc_offset = LOWER_EL_AArch64_VECTOR; 107 else 108 exc_offset = LOWER_EL_AArch32_VECTOR; 109 110 switch (target_mode) { 111 case PSR_MODE_EL1h: 112 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1); 113 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 114 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); 115 break; 116 case PSR_MODE_EL2h: 117 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2); 118 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2); 119 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2); 120 break; 121 default: 122 /* Don't do that */ 123 BUG(); 124 } 125 126 *vcpu_pc(vcpu) = vbar + exc_offset + type; 127 128 old = *vcpu_cpsr(vcpu); 129 new = 0; 130 131 new |= (old & PSR_N_BIT); 132 new |= (old & PSR_Z_BIT); 133 new |= (old & PSR_C_BIT); 134 new |= (old & PSR_V_BIT); 135 136 if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) 137 new |= PSR_TCO_BIT; 138 139 new |= (old & PSR_DIT_BIT); 140 141 // PSTATE.UAO is set to zero upon any exception to AArch64 142 // See ARM DDI 0487E.a, page D5-2579. 143 144 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0 145 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented 146 // See ARM DDI 0487E.a, page D5-2578. 147 new |= (old & PSR_PAN_BIT); 148 if (!(sctlr & SCTLR_EL1_SPAN)) 149 new |= PSR_PAN_BIT; 150 151 // PSTATE.SS is set to zero upon any exception to AArch64 152 // See ARM DDI 0487E.a, page D2-2452. 153 154 // PSTATE.IL is set to zero upon any exception to AArch64 155 // See ARM DDI 0487E.a, page D1-2306. 156 157 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64 158 // See ARM DDI 0487E.a, page D13-3258 159 if (sctlr & SCTLR_ELx_DSSBS) 160 new |= PSR_SSBS_BIT; 161 162 // PSTATE.BTYPE is set to zero upon any exception to AArch64 163 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294. 164 165 new |= PSR_D_BIT; 166 new |= PSR_A_BIT; 167 new |= PSR_I_BIT; 168 new |= PSR_F_BIT; 169 170 new |= target_mode; 171 172 *vcpu_cpsr(vcpu) = new; 173 __vcpu_write_spsr(vcpu, target_mode, old); 174 } 175 176 /* 177 * When an exception is taken, most CPSR fields are left unchanged in the 178 * handler. However, some are explicitly overridden (e.g. M[4:0]). 179 * 180 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with 181 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was 182 * obsoleted by the ARMv7 virtualization extensions and is RES0. 183 * 184 * For the SPSR layout seen from AArch32, see: 185 * - ARM DDI 0406C.d, page B1-1148 186 * - ARM DDI 0487E.a, page G8-6264 187 * 188 * For the SPSR_ELx layout for AArch32 seen from AArch64, see: 189 * - ARM DDI 0487E.a, page C5-426 190 * 191 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from 192 * MSB to LSB. 193 */ 194 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) 195 { 196 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 197 unsigned long old, new; 198 199 old = *vcpu_cpsr(vcpu); 200 new = 0; 201 202 new |= (old & PSR_AA32_N_BIT); 203 new |= (old & PSR_AA32_Z_BIT); 204 new |= (old & PSR_AA32_C_BIT); 205 new |= (old & PSR_AA32_V_BIT); 206 new |= (old & PSR_AA32_Q_BIT); 207 208 // CPSR.IT[7:0] are set to zero upon any exception 209 // See ARM DDI 0487E.a, section G1.12.3 210 // See ARM DDI 0406C.d, section B1.8.3 211 212 new |= (old & PSR_AA32_DIT_BIT); 213 214 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception 215 // See ARM DDI 0487E.a, page G8-6244 216 if (sctlr & BIT(31)) 217 new |= PSR_AA32_SSBS_BIT; 218 219 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0 220 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented 221 // See ARM DDI 0487E.a, page G8-6246 222 new |= (old & PSR_AA32_PAN_BIT); 223 if (!(sctlr & BIT(23))) 224 new |= PSR_AA32_PAN_BIT; 225 226 // SS does not exist in AArch32, so ignore 227 228 // CPSR.IL is set to zero upon any exception 229 // See ARM DDI 0487E.a, page G1-5527 230 231 new |= (old & PSR_AA32_GE_MASK); 232 233 // CPSR.IT[7:0] are set to zero upon any exception 234 // See prior comment above 235 236 // CPSR.E is set to SCTLR.EE upon any exception 237 // See ARM DDI 0487E.a, page G8-6245 238 // See ARM DDI 0406C.d, page B4-1701 239 if (sctlr & BIT(25)) 240 new |= PSR_AA32_E_BIT; 241 242 // CPSR.A is unchanged upon an exception to Undefined, Supervisor 243 // CPSR.A is set upon an exception to other modes 244 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 245 // See ARM DDI 0406C.d, page B1-1182 246 new |= (old & PSR_AA32_A_BIT); 247 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC) 248 new |= PSR_AA32_A_BIT; 249 250 // CPSR.I is set upon any exception 251 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 252 // See ARM DDI 0406C.d, page B1-1182 253 new |= PSR_AA32_I_BIT; 254 255 // CPSR.F is set upon an exception to FIQ 256 // CPSR.F is unchanged upon an exception to other modes 257 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516 258 // See ARM DDI 0406C.d, page B1-1182 259 new |= (old & PSR_AA32_F_BIT); 260 if (mode == PSR_AA32_MODE_FIQ) 261 new |= PSR_AA32_F_BIT; 262 263 // CPSR.T is set to SCTLR.TE upon any exception 264 // See ARM DDI 0487E.a, page G8-5514 265 // See ARM DDI 0406C.d, page B1-1181 266 if (sctlr & BIT(30)) 267 new |= PSR_AA32_T_BIT; 268 269 new |= mode; 270 271 return new; 272 } 273 274 /* 275 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10. 276 */ 277 static const u8 return_offsets[8][2] = { 278 [0] = { 0, 0 }, /* Reset, unused */ 279 [1] = { 4, 2 }, /* Undefined */ 280 [2] = { 0, 0 }, /* SVC, unused */ 281 [3] = { 4, 4 }, /* Prefetch abort */ 282 [4] = { 8, 8 }, /* Data abort */ 283 [5] = { 0, 0 }, /* HVC, unused */ 284 [6] = { 4, 4 }, /* IRQ, unused */ 285 [7] = { 4, 4 }, /* FIQ, unused */ 286 }; 287 288 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) 289 { 290 unsigned long spsr = *vcpu_cpsr(vcpu); 291 bool is_thumb = (spsr & PSR_AA32_T_BIT); 292 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); 293 u32 return_address; 294 295 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); 296 return_address = *vcpu_pc(vcpu); 297 return_address += return_offsets[vect_offset >> 2][is_thumb]; 298 299 /* KVM only enters the ABT and UND modes, so only deal with those */ 300 switch(mode) { 301 case PSR_AA32_MODE_ABT: 302 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); 303 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; 304 break; 305 306 case PSR_AA32_MODE_UND: 307 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); 308 vcpu_gp_regs(vcpu)->compat_lr_und = return_address; 309 break; 310 } 311 312 /* Branch to exception vector */ 313 if (sctlr & (1 << 13)) 314 vect_offset += 0xffff0000; 315 else /* always have security exceptions */ 316 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1); 317 318 *vcpu_pc(vcpu) = vect_offset; 319 } 320 321 static void kvm_inject_exception(struct kvm_vcpu *vcpu) 322 { 323 if (vcpu_el1_is_32bit(vcpu)) { 324 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) { 325 case unpack_vcpu_flag(EXCEPT_AA32_UND): 326 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4); 327 break; 328 case unpack_vcpu_flag(EXCEPT_AA32_IABT): 329 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12); 330 break; 331 case unpack_vcpu_flag(EXCEPT_AA32_DABT): 332 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16); 333 break; 334 default: 335 /* Err... */ 336 break; 337 } 338 } else { 339 switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) { 340 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC): 341 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync); 342 break; 343 344 case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR): 345 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_serror); 346 break; 347 348 case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC): 349 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync); 350 break; 351 352 case unpack_vcpu_flag(EXCEPT_AA64_EL2_IRQ): 353 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq); 354 break; 355 356 case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR): 357 enter_exception64(vcpu, PSR_MODE_EL2h, except_type_serror); 358 break; 359 360 default: 361 /* 362 * Only EL1_{SYNC,SERR} and EL2_{SYNC,IRQ,SERR} makes 363 * sense so far. Everything else gets silently 364 * ignored. 365 */ 366 break; 367 } 368 } 369 } 370 371 /* 372 * Adjust the guest PC (and potentially exception state) depending on 373 * flags provided by the emulation code. 374 */ 375 void __kvm_adjust_pc(struct kvm_vcpu *vcpu) 376 { 377 if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) { 378 kvm_inject_exception(vcpu); 379 vcpu_clear_flag(vcpu, PENDING_EXCEPTION); 380 vcpu_clear_flag(vcpu, EXCEPT_MASK); 381 } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) { 382 kvm_skip_instr(vcpu); 383 vcpu_clear_flag(vcpu, INCREMENT_PC); 384 } 385 } 386