1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ARM64_KVM_NESTED_H 3 #define __ARM64_KVM_NESTED_H 4 5 #include <linux/bitfield.h> 6 #include <linux/kvm_host.h> 7 #include <asm/kvm_emulate.h> 8 #include <asm/kvm_pgtable.h> 9 10 static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) 11 { 12 return (!__is_defined(__KVM_NVHE_HYPERVISOR__) && 13 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && 14 vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2)); 15 } 16 17 /* Translation helpers from non-VHE EL2 to EL1 */ 18 static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2) 19 { 20 return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT; 21 } 22 23 static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr) 24 { 25 return TCR_EPD1_MASK | /* disable TTBR1_EL1 */ 26 ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) | 27 tcr_el2_ps_to_tcr_el1_ips(tcr) | 28 (tcr & TCR_EL2_TG0_MASK) | 29 (tcr & TCR_EL2_ORGN0_MASK) | 30 (tcr & TCR_EL2_IRGN0_MASK) | 31 (tcr & TCR_EL2_T0SZ_MASK); 32 } 33 34 static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2) 35 { 36 u64 cpacr_el1 = CPACR_EL1_RES1; 37 38 if (cptr_el2 & CPTR_EL2_TTA) 39 cpacr_el1 |= CPACR_EL1_TTA; 40 if (!(cptr_el2 & CPTR_EL2_TFP)) 41 cpacr_el1 |= CPACR_EL1_FPEN; 42 if (!(cptr_el2 & CPTR_EL2_TZ)) 43 cpacr_el1 |= CPACR_EL1_ZEN; 44 45 cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM); 46 47 return cpacr_el1; 48 } 49 50 static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val) 51 { 52 /* Only preserve the minimal set of bits we support */ 53 val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA | 54 SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE); 55 val |= SCTLR_EL1_RES1; 56 57 return val; 58 } 59 60 static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) 61 { 62 /* Clear the ASID field */ 63 return ttbr0 & ~GENMASK_ULL(63, 48); 64 } 65 66 extern bool forward_smc_trap(struct kvm_vcpu *vcpu); 67 extern bool forward_debug_exception(struct kvm_vcpu *vcpu); 68 extern void kvm_init_nested(struct kvm *kvm); 69 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); 70 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); 71 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); 72 73 union tlbi_info; 74 75 extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 76 const union tlbi_info *info, 77 void (*)(struct kvm_s2_mmu *, 78 const union tlbi_info *)); 79 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); 80 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); 81 82 extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu); 83 84 struct kvm_s2_trans { 85 phys_addr_t output; 86 unsigned long block_size; 87 bool writable; 88 bool readable; 89 int level; 90 u32 esr; 91 u64 desc; 92 }; 93 94 static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans) 95 { 96 return trans->output; 97 } 98 99 static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans) 100 { 101 return trans->block_size; 102 } 103 104 static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans) 105 { 106 return trans->esr; 107 } 108 109 static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans) 110 { 111 return trans->readable; 112 } 113 114 static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans) 115 { 116 return trans->writable; 117 } 118 119 static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans) 120 { 121 return !(trans->desc & BIT(54)); 122 } 123 124 extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 125 struct kvm_s2_trans *result); 126 extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, 127 struct kvm_s2_trans *trans); 128 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); 129 extern void kvm_nested_s2_wp(struct kvm *kvm); 130 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); 131 extern void kvm_nested_s2_flush(struct kvm *kvm); 132 133 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val); 134 135 static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr) 136 { 137 struct kvm *kvm = vpcu->kvm; 138 u8 CRm = sys_reg_CRm(instr); 139 140 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 141 sys_reg_Op1(instr) == TLBI_Op1_EL1)) 142 return false; 143 144 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 145 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 146 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 147 return false; 148 149 if (CRm == TLBI_CRm_nROS && 150 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 151 return false; 152 153 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 154 CRm == TLBI_CRm_RNS) && 155 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 156 return false; 157 158 return true; 159 } 160 161 static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr) 162 { 163 struct kvm *kvm = vpcu->kvm; 164 u8 CRm = sys_reg_CRm(instr); 165 166 if (!(sys_reg_Op0(instr) == TLBI_Op0 && 167 sys_reg_Op1(instr) == TLBI_Op1_EL2)) 168 return false; 169 170 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS || 171 (sys_reg_CRn(instr) == TLBI_CRn_nXS && 172 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)))) 173 return false; 174 175 if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS) 176 return false; 177 178 if (CRm == TLBI_CRm_nROS && 179 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 180 return false; 181 182 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS || 183 CRm == TLBI_CRm_RNS) && 184 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 185 return false; 186 187 return true; 188 } 189 190 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu); 191 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val); 192 193 #ifdef CONFIG_ARM64_PTR_AUTH 194 bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr); 195 #else 196 static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr) 197 { 198 /* We really should never execute this... */ 199 WARN_ON_ONCE(1); 200 *elr = 0xbad9acc0debadbad; 201 return false; 202 } 203 #endif 204 205 #define KVM_NV_GUEST_MAP_SZ (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0) 206 207 static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) 208 { 209 return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level); 210 } 211 212 /* Adjust alignment for the contiguous bit as per StageOA() */ 213 #define contiguous_bit_shift(d, wi, l) \ 214 ({ \ 215 u8 shift = 0; \ 216 \ 217 if ((d) & PTE_CONT) { \ 218 switch (BIT((wi)->pgshift)) { \ 219 case SZ_4K: \ 220 shift = 4; \ 221 break; \ 222 case SZ_16K: \ 223 shift = (l) == 2 ? 5 : 7; \ 224 break; \ 225 case SZ_64K: \ 226 shift = 5; \ 227 break; \ 228 } \ 229 } \ 230 \ 231 shift; \ 232 }) 233 234 static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) 235 { 236 u64 base, tg, num, scale; 237 int shift; 238 239 tg = FIELD_GET(GENMASK(47, 46), val); 240 241 switch(tg) { 242 case 1: 243 shift = 12; 244 break; 245 case 2: 246 shift = 14; 247 break; 248 case 3: 249 default: /* IMPDEF: handle tg==0 as 64k */ 250 shift = 16; 251 break; 252 } 253 254 base = (val & GENMASK(36, 0)) << shift; 255 256 if (asid) 257 *asid = FIELD_GET(TLBIR_ASID_MASK, val); 258 259 scale = FIELD_GET(GENMASK(45, 44), val); 260 num = FIELD_GET(GENMASK(43, 39), val); 261 *range = __TLBI_RANGE_PAGES(num, scale) << shift; 262 263 return base; 264 } 265 266 static inline unsigned int ps_to_output_size(unsigned int ps) 267 { 268 switch (ps) { 269 case 0: return 32; 270 case 1: return 36; 271 case 2: return 40; 272 case 3: return 42; 273 case 4: return 44; 274 case 5: 275 default: 276 return 48; 277 } 278 } 279 280 enum trans_regime { 281 TR_EL10, 282 TR_EL20, 283 TR_EL2, 284 }; 285 286 struct s1_walk_info { 287 u64 baddr; 288 enum trans_regime regime; 289 unsigned int max_oa_bits; 290 unsigned int pgshift; 291 unsigned int txsz; 292 int sl; 293 bool as_el0; 294 bool hpd; 295 bool e0poe; 296 bool poe; 297 bool pan; 298 bool be; 299 bool s2; 300 }; 301 302 struct s1_walk_result { 303 union { 304 struct { 305 u64 desc; 306 u64 pa; 307 s8 level; 308 u8 APTable; 309 bool nG; 310 u16 asid; 311 bool UXNTable; 312 bool PXNTable; 313 bool uwxn; 314 bool uov; 315 bool ur; 316 bool uw; 317 bool ux; 318 bool pwxn; 319 bool pov; 320 bool pr; 321 bool pw; 322 bool px; 323 }; 324 struct { 325 u8 fst; 326 bool ptw; 327 bool s2; 328 }; 329 }; 330 bool failed; 331 }; 332 333 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, 334 struct s1_walk_result *wr, u64 va); 335 336 /* VNCR management */ 337 int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu); 338 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu); 339 void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val); 340 341 #define vncr_fixmap(c) \ 342 ({ \ 343 u32 __c = (c); \ 344 BUG_ON(__c >= NR_CPUS); \ 345 (FIX_VNCR - __c); \ 346 }) 347 348 #endif /* __ARM64_KVM_NESTED_H */ 349