1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 - Columbia University and Linaro Ltd. 4 * Author: Jintack Lim <jintack.lim@linaro.org> 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/kvm.h> 9 #include <linux/kvm_host.h> 10 11 #include <asm/kvm_arm.h> 12 #include <asm/kvm_emulate.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/kvm_nested.h> 15 #include <asm/sysreg.h> 16 17 #include "sys_regs.h" 18 19 /* 20 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between 21 * memory usage and potential number of different sets of S2 PTs in 22 * the guests. Running out of S2 MMUs only affects performance (we 23 * will invalidate them more often). 24 */ 25 #define S2_MMU_PER_VCPU 2 26 27 void kvm_init_nested(struct kvm *kvm) 28 { 29 kvm->arch.nested_mmus = NULL; 30 kvm->arch.nested_mmus_size = 0; 31 } 32 33 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) 34 { 35 /* 36 * We only initialise the IPA range on the canonical MMU, which 37 * defines the contract between KVM and userspace on where the 38 * "hardware" is in the IPA space. This affects the validity of MMIO 39 * exits forwarded to userspace, for example. 40 * 41 * For nested S2s, we use the PARange as exposed to the guest, as it 42 * is allowed to use it at will to expose whatever memory map it 43 * wants to its own guests as it would be on real HW. 44 */ 45 return kvm_init_stage2_mmu(kvm, mmu, kvm_get_pa_bits(kvm)); 46 } 47 48 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) 49 { 50 struct kvm *kvm = vcpu->kvm; 51 struct kvm_s2_mmu *tmp; 52 int num_mmus, ret = 0; 53 54 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && 55 !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) 56 return -EINVAL; 57 58 /* 59 * Let's treat memory allocation failures as benign: If we fail to 60 * allocate anything, return an error and keep the allocated array 61 * alive. Userspace may try to recover by intializing the vcpu 62 * again, and there is no reason to affect the whole VM for this. 63 */ 64 num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; 65 tmp = kvrealloc(kvm->arch.nested_mmus, 66 size_mul(sizeof(*kvm->arch.nested_mmus), num_mmus), 67 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 68 if (!tmp) 69 return -ENOMEM; 70 71 swap(kvm->arch.nested_mmus, tmp); 72 73 /* 74 * If we went through a realocation, adjust the MMU back-pointers in 75 * the previously initialised kvm_pgtable structures. 76 */ 77 if (kvm->arch.nested_mmus != tmp) 78 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) 79 kvm->arch.nested_mmus[i].pgt->mmu = &kvm->arch.nested_mmus[i]; 80 81 for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) 82 ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); 83 84 if (ret) { 85 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) 86 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); 87 88 return ret; 89 } 90 91 kvm->arch.nested_mmus_size = num_mmus; 92 93 return 0; 94 } 95 96 struct s2_walk_info { 97 int (*read_desc)(phys_addr_t pa, u64 *desc, void *data); 98 void *data; 99 u64 baddr; 100 unsigned int max_oa_bits; 101 unsigned int pgshift; 102 unsigned int sl; 103 unsigned int t0sz; 104 bool be; 105 }; 106 107 static u32 compute_fsc(int level, u32 fsc) 108 { 109 return fsc | (level & 0x3); 110 } 111 112 static int esr_s2_fault(struct kvm_vcpu *vcpu, int level, u32 fsc) 113 { 114 u32 esr; 115 116 esr = kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC; 117 esr |= compute_fsc(level, fsc); 118 return esr; 119 } 120 121 static int get_ia_size(struct s2_walk_info *wi) 122 { 123 return 64 - wi->t0sz; 124 } 125 126 static int check_base_s2_limits(struct s2_walk_info *wi, 127 int level, int input_size, int stride) 128 { 129 int start_size, ia_size; 130 131 ia_size = get_ia_size(wi); 132 133 /* Check translation limits */ 134 switch (BIT(wi->pgshift)) { 135 case SZ_64K: 136 if (level == 0 || (level == 1 && ia_size <= 42)) 137 return -EFAULT; 138 break; 139 case SZ_16K: 140 if (level == 0 || (level == 1 && ia_size <= 40)) 141 return -EFAULT; 142 break; 143 case SZ_4K: 144 if (level < 0 || (level == 0 && ia_size <= 42)) 145 return -EFAULT; 146 break; 147 } 148 149 /* Check input size limits */ 150 if (input_size > ia_size) 151 return -EFAULT; 152 153 /* Check number of entries in starting level table */ 154 start_size = input_size - ((3 - level) * stride + wi->pgshift); 155 if (start_size < 1 || start_size > stride + 4) 156 return -EFAULT; 157 158 return 0; 159 } 160 161 /* Check if output is within boundaries */ 162 static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) 163 { 164 unsigned int output_size = wi->max_oa_bits; 165 166 if (output_size != 48 && (output & GENMASK_ULL(47, output_size))) 167 return -1; 168 169 return 0; 170 } 171 172 /* 173 * This is essentially a C-version of the pseudo code from the ARM ARM 174 * AArch64.TranslationTableWalk function. I strongly recommend looking at 175 * that pseudocode in trying to understand this. 176 * 177 * Must be called with the kvm->srcu read lock held 178 */ 179 static int walk_nested_s2_pgd(phys_addr_t ipa, 180 struct s2_walk_info *wi, struct kvm_s2_trans *out) 181 { 182 int first_block_level, level, stride, input_size, base_lower_bound; 183 phys_addr_t base_addr; 184 unsigned int addr_top, addr_bottom; 185 u64 desc; /* page table entry */ 186 int ret; 187 phys_addr_t paddr; 188 189 switch (BIT(wi->pgshift)) { 190 default: 191 case SZ_64K: 192 case SZ_16K: 193 level = 3 - wi->sl; 194 first_block_level = 2; 195 break; 196 case SZ_4K: 197 level = 2 - wi->sl; 198 first_block_level = 1; 199 break; 200 } 201 202 stride = wi->pgshift - 3; 203 input_size = get_ia_size(wi); 204 if (input_size > 48 || input_size < 25) 205 return -EFAULT; 206 207 ret = check_base_s2_limits(wi, level, input_size, stride); 208 if (WARN_ON(ret)) 209 return ret; 210 211 base_lower_bound = 3 + input_size - ((3 - level) * stride + 212 wi->pgshift); 213 base_addr = wi->baddr & GENMASK_ULL(47, base_lower_bound); 214 215 if (check_output_size(wi, base_addr)) { 216 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 217 return 1; 218 } 219 220 addr_top = input_size - 1; 221 222 while (1) { 223 phys_addr_t index; 224 225 addr_bottom = (3 - level) * stride + wi->pgshift; 226 index = (ipa & GENMASK_ULL(addr_top, addr_bottom)) 227 >> (addr_bottom - 3); 228 229 paddr = base_addr | index; 230 ret = wi->read_desc(paddr, &desc, wi->data); 231 if (ret < 0) 232 return ret; 233 234 /* 235 * Handle reversedescriptors if endianness differs between the 236 * host and the guest hypervisor. 237 */ 238 if (wi->be) 239 desc = be64_to_cpu((__force __be64)desc); 240 else 241 desc = le64_to_cpu((__force __le64)desc); 242 243 /* Check for valid descriptor at this point */ 244 if (!(desc & 1) || ((desc & 3) == 1 && level == 3)) { 245 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 246 out->desc = desc; 247 return 1; 248 } 249 250 /* We're at the final level or block translation level */ 251 if ((desc & 3) == 1 || level == 3) 252 break; 253 254 if (check_output_size(wi, desc)) { 255 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 256 out->desc = desc; 257 return 1; 258 } 259 260 base_addr = desc & GENMASK_ULL(47, wi->pgshift); 261 262 level += 1; 263 addr_top = addr_bottom - 1; 264 } 265 266 if (level < first_block_level) { 267 out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT); 268 out->desc = desc; 269 return 1; 270 } 271 272 if (check_output_size(wi, desc)) { 273 out->esr = compute_fsc(level, ESR_ELx_FSC_ADDRSZ); 274 out->desc = desc; 275 return 1; 276 } 277 278 if (!(desc & BIT(10))) { 279 out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS); 280 out->desc = desc; 281 return 1; 282 } 283 284 addr_bottom += contiguous_bit_shift(desc, wi, level); 285 286 /* Calculate and return the result */ 287 paddr = (desc & GENMASK_ULL(47, addr_bottom)) | 288 (ipa & GENMASK_ULL(addr_bottom - 1, 0)); 289 out->output = paddr; 290 out->block_size = 1UL << ((3 - level) * stride + wi->pgshift); 291 out->readable = desc & (0b01 << 6); 292 out->writable = desc & (0b10 << 6); 293 out->level = level; 294 out->desc = desc; 295 return 0; 296 } 297 298 static int read_guest_s2_desc(phys_addr_t pa, u64 *desc, void *data) 299 { 300 struct kvm_vcpu *vcpu = data; 301 302 return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc)); 303 } 304 305 static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi) 306 { 307 wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK; 308 309 switch (vtcr & VTCR_EL2_TG0_MASK) { 310 case VTCR_EL2_TG0_4K: 311 wi->pgshift = 12; break; 312 case VTCR_EL2_TG0_16K: 313 wi->pgshift = 14; break; 314 case VTCR_EL2_TG0_64K: 315 default: /* IMPDEF: treat any other value as 64k */ 316 wi->pgshift = 16; break; 317 } 318 319 wi->sl = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); 320 /* Global limit for now, should eventually be per-VM */ 321 wi->max_oa_bits = min(get_kvm_ipa_limit(), 322 ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr))); 323 } 324 325 int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, 326 struct kvm_s2_trans *result) 327 { 328 u64 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 329 struct s2_walk_info wi; 330 int ret; 331 332 result->esr = 0; 333 334 if (!vcpu_has_nv(vcpu)) 335 return 0; 336 337 wi.read_desc = read_guest_s2_desc; 338 wi.data = vcpu; 339 wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 340 341 vtcr_to_walk_info(vtcr, &wi); 342 343 wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE; 344 345 ret = walk_nested_s2_pgd(gipa, &wi, result); 346 if (ret) 347 result->esr |= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC); 348 349 return ret; 350 } 351 352 static unsigned int ttl_to_size(u8 ttl) 353 { 354 int level = ttl & 3; 355 int gran = (ttl >> 2) & 3; 356 unsigned int max_size = 0; 357 358 switch (gran) { 359 case TLBI_TTL_TG_4K: 360 switch (level) { 361 case 0: 362 break; 363 case 1: 364 max_size = SZ_1G; 365 break; 366 case 2: 367 max_size = SZ_2M; 368 break; 369 case 3: 370 max_size = SZ_4K; 371 break; 372 } 373 break; 374 case TLBI_TTL_TG_16K: 375 switch (level) { 376 case 0: 377 case 1: 378 break; 379 case 2: 380 max_size = SZ_32M; 381 break; 382 case 3: 383 max_size = SZ_16K; 384 break; 385 } 386 break; 387 case TLBI_TTL_TG_64K: 388 switch (level) { 389 case 0: 390 case 1: 391 /* No 52bit IPA support */ 392 break; 393 case 2: 394 max_size = SZ_512M; 395 break; 396 case 3: 397 max_size = SZ_64K; 398 break; 399 } 400 break; 401 default: /* No size information */ 402 break; 403 } 404 405 return max_size; 406 } 407 408 /* 409 * Compute the equivalent of the TTL field by parsing the shadow PT. The 410 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is 411 * retrieved from first entry carrying the level as a tag. 412 */ 413 static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) 414 { 415 u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr; 416 kvm_pte_t pte; 417 u8 ttl, level; 418 419 lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock); 420 421 switch (vtcr & VTCR_EL2_TG0_MASK) { 422 case VTCR_EL2_TG0_4K: 423 ttl = (TLBI_TTL_TG_4K << 2); 424 break; 425 case VTCR_EL2_TG0_16K: 426 ttl = (TLBI_TTL_TG_16K << 2); 427 break; 428 case VTCR_EL2_TG0_64K: 429 default: /* IMPDEF: treat any other value as 64k */ 430 ttl = (TLBI_TTL_TG_64K << 2); 431 break; 432 } 433 434 tmp = addr; 435 436 again: 437 /* Iteratively compute the block sizes for a particular granule size */ 438 switch (vtcr & VTCR_EL2_TG0_MASK) { 439 case VTCR_EL2_TG0_4K: 440 if (sz < SZ_4K) sz = SZ_4K; 441 else if (sz < SZ_2M) sz = SZ_2M; 442 else if (sz < SZ_1G) sz = SZ_1G; 443 else sz = 0; 444 break; 445 case VTCR_EL2_TG0_16K: 446 if (sz < SZ_16K) sz = SZ_16K; 447 else if (sz < SZ_32M) sz = SZ_32M; 448 else sz = 0; 449 break; 450 case VTCR_EL2_TG0_64K: 451 default: /* IMPDEF: treat any other value as 64k */ 452 if (sz < SZ_64K) sz = SZ_64K; 453 else if (sz < SZ_512M) sz = SZ_512M; 454 else sz = 0; 455 break; 456 } 457 458 if (sz == 0) 459 return 0; 460 461 tmp &= ~(sz - 1); 462 if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) 463 goto again; 464 if (!(pte & PTE_VALID)) 465 goto again; 466 level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte); 467 if (!level) 468 goto again; 469 470 ttl |= level; 471 472 /* 473 * We now have found some level information in the shadow S2. Check 474 * that the resulting range is actually including the original IPA. 475 */ 476 sz = ttl_to_size(ttl); 477 if (addr < (tmp + sz)) 478 return ttl; 479 480 return 0; 481 } 482 483 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) 484 { 485 struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); 486 unsigned long max_size; 487 u8 ttl; 488 489 ttl = FIELD_GET(TLBI_TTL_MASK, val); 490 491 if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) { 492 /* No TTL, check the shadow S2 for a hint */ 493 u64 addr = (val & GENMASK_ULL(35, 0)) << 12; 494 ttl = get_guest_mapping_ttl(mmu, addr); 495 } 496 497 max_size = ttl_to_size(ttl); 498 499 if (!max_size) { 500 /* Compute the maximum extent of the invalidation */ 501 switch (mmu->tlb_vtcr & VTCR_EL2_TG0_MASK) { 502 case VTCR_EL2_TG0_4K: 503 max_size = SZ_1G; 504 break; 505 case VTCR_EL2_TG0_16K: 506 max_size = SZ_32M; 507 break; 508 case VTCR_EL2_TG0_64K: 509 default: /* IMPDEF: treat any other value as 64k */ 510 /* 511 * No, we do not support 52bit IPA in nested yet. Once 512 * we do, this should be 4TB. 513 */ 514 max_size = SZ_512M; 515 break; 516 } 517 } 518 519 WARN_ON(!max_size); 520 return max_size; 521 } 522 523 /* 524 * We can have multiple *different* MMU contexts with the same VMID: 525 * 526 * - S2 being enabled or not, hence differing by the HCR_EL2.VM bit 527 * 528 * - Multiple vcpus using private S2s (huh huh...), hence differing by the 529 * VBBTR_EL2.BADDR address 530 * 531 * - A combination of the above... 532 * 533 * We can always identify which MMU context to pick at run-time. However, 534 * TLB invalidation involving a VMID must take action on all the TLBs using 535 * this particular VMID. This translates into applying the same invalidation 536 * operation to all the contexts that are using this VMID. Moar phun! 537 */ 538 void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, 539 const union tlbi_info *info, 540 void (*tlbi_callback)(struct kvm_s2_mmu *, 541 const union tlbi_info *)) 542 { 543 write_lock(&kvm->mmu_lock); 544 545 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 546 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 547 548 if (!kvm_s2_mmu_valid(mmu)) 549 continue; 550 551 if (vmid == get_vmid(mmu->tlb_vttbr)) 552 tlbi_callback(mmu, info); 553 } 554 555 write_unlock(&kvm->mmu_lock); 556 } 557 558 struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) 559 { 560 struct kvm *kvm = vcpu->kvm; 561 bool nested_stage2_enabled; 562 u64 vttbr, vtcr, hcr; 563 564 lockdep_assert_held_write(&kvm->mmu_lock); 565 566 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); 567 vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 568 hcr = vcpu_read_sys_reg(vcpu, HCR_EL2); 569 570 nested_stage2_enabled = hcr & HCR_VM; 571 572 /* Don't consider the CnP bit for the vttbr match */ 573 vttbr &= ~VTTBR_CNP_BIT; 574 575 /* 576 * Two possibilities when looking up a S2 MMU context: 577 * 578 * - either S2 is enabled in the guest, and we need a context that is 579 * S2-enabled and matches the full VTTBR (VMID+BADDR) and VTCR, 580 * which makes it safe from a TLB conflict perspective (a broken 581 * guest won't be able to generate them), 582 * 583 * - or S2 is disabled, and we need a context that is S2-disabled 584 * and matches the VMID only, as all TLBs are tagged by VMID even 585 * if S2 translation is disabled. 586 */ 587 for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { 588 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 589 590 if (!kvm_s2_mmu_valid(mmu)) 591 continue; 592 593 if (nested_stage2_enabled && 594 mmu->nested_stage2_enabled && 595 vttbr == mmu->tlb_vttbr && 596 vtcr == mmu->tlb_vtcr) 597 return mmu; 598 599 if (!nested_stage2_enabled && 600 !mmu->nested_stage2_enabled && 601 get_vmid(vttbr) == get_vmid(mmu->tlb_vttbr)) 602 return mmu; 603 } 604 return NULL; 605 } 606 607 static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) 608 { 609 struct kvm *kvm = vcpu->kvm; 610 struct kvm_s2_mmu *s2_mmu; 611 int i; 612 613 lockdep_assert_held_write(&vcpu->kvm->mmu_lock); 614 615 s2_mmu = lookup_s2_mmu(vcpu); 616 if (s2_mmu) 617 goto out; 618 619 /* 620 * Make sure we don't always search from the same point, or we 621 * will always reuse a potentially active context, leaving 622 * free contexts unused. 623 */ 624 for (i = kvm->arch.nested_mmus_next; 625 i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); 626 i++) { 627 s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; 628 629 if (atomic_read(&s2_mmu->refcnt) == 0) 630 break; 631 } 632 BUG_ON(atomic_read(&s2_mmu->refcnt)); /* We have struct MMUs to spare */ 633 634 /* Set the scene for the next search */ 635 kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; 636 637 /* Make sure we don't forget to do the laundry */ 638 if (kvm_s2_mmu_valid(s2_mmu)) 639 s2_mmu->pending_unmap = true; 640 641 /* 642 * The virtual VMID (modulo CnP) will be used as a key when matching 643 * an existing kvm_s2_mmu. 644 * 645 * We cache VTCR at allocation time, once and for all. It'd be great 646 * if the guest didn't screw that one up, as this is not very 647 * forgiving... 648 */ 649 s2_mmu->tlb_vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2) & ~VTTBR_CNP_BIT; 650 s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); 651 s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; 652 653 out: 654 atomic_inc(&s2_mmu->refcnt); 655 656 /* 657 * Set the vCPU request to perform an unmap, even if the pending unmap 658 * originates from another vCPU. This guarantees that the MMU has been 659 * completely unmapped before any vCPU actually uses it, and allows 660 * multiple vCPUs to lend a hand with completing the unmap. 661 */ 662 if (s2_mmu->pending_unmap) 663 kvm_make_request(KVM_REQ_NESTED_S2_UNMAP, vcpu); 664 665 return s2_mmu; 666 } 667 668 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) 669 { 670 /* CnP being set denotes an invalid entry */ 671 mmu->tlb_vttbr = VTTBR_CNP_BIT; 672 mmu->nested_stage2_enabled = false; 673 atomic_set(&mmu->refcnt, 0); 674 } 675 676 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) 677 { 678 /* 679 * The vCPU kept its reference on the MMU after the last put, keep 680 * rolling with it. 681 */ 682 if (vcpu->arch.hw_mmu) 683 return; 684 685 if (is_hyp_ctxt(vcpu)) { 686 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; 687 } else { 688 write_lock(&vcpu->kvm->mmu_lock); 689 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); 690 write_unlock(&vcpu->kvm->mmu_lock); 691 } 692 } 693 694 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) 695 { 696 /* 697 * Keep a reference on the associated stage-2 MMU if the vCPU is 698 * scheduling out and not in WFI emulation, suggesting it is likely to 699 * reuse the MMU sometime soon. 700 */ 701 if (vcpu->scheduled_out && !vcpu_get_flag(vcpu, IN_WFI)) 702 return; 703 704 if (kvm_is_nested_s2_mmu(vcpu->kvm, vcpu->arch.hw_mmu)) 705 atomic_dec(&vcpu->arch.hw_mmu->refcnt); 706 707 vcpu->arch.hw_mmu = NULL; 708 } 709 710 /* 711 * Returns non-zero if permission fault is handled by injecting it to the next 712 * level hypervisor. 713 */ 714 int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans) 715 { 716 bool forward_fault = false; 717 718 trans->esr = 0; 719 720 if (!kvm_vcpu_trap_is_permission_fault(vcpu)) 721 return 0; 722 723 if (kvm_vcpu_trap_is_iabt(vcpu)) { 724 forward_fault = !kvm_s2_trans_executable(trans); 725 } else { 726 bool write_fault = kvm_is_write_fault(vcpu); 727 728 forward_fault = ((write_fault && !trans->writable) || 729 (!write_fault && !trans->readable)); 730 } 731 732 if (forward_fault) 733 trans->esr = esr_s2_fault(vcpu, trans->level, ESR_ELx_FSC_PERM); 734 735 return forward_fault; 736 } 737 738 int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) 739 { 740 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.far_el2, FAR_EL2); 741 vcpu_write_sys_reg(vcpu, vcpu->arch.fault.hpfar_el2, HPFAR_EL2); 742 743 return kvm_inject_nested_sync(vcpu, esr_el2); 744 } 745 746 void kvm_nested_s2_wp(struct kvm *kvm) 747 { 748 int i; 749 750 lockdep_assert_held_write(&kvm->mmu_lock); 751 752 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 753 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 754 755 if (kvm_s2_mmu_valid(mmu)) 756 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); 757 } 758 } 759 760 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) 761 { 762 int i; 763 764 lockdep_assert_held_write(&kvm->mmu_lock); 765 766 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 767 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 768 769 if (kvm_s2_mmu_valid(mmu)) 770 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); 771 } 772 } 773 774 void kvm_nested_s2_flush(struct kvm *kvm) 775 { 776 int i; 777 778 lockdep_assert_held_write(&kvm->mmu_lock); 779 780 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 781 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 782 783 if (kvm_s2_mmu_valid(mmu)) 784 kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); 785 } 786 } 787 788 void kvm_arch_flush_shadow_all(struct kvm *kvm) 789 { 790 int i; 791 792 for (i = 0; i < kvm->arch.nested_mmus_size; i++) { 793 struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; 794 795 if (!WARN_ON(atomic_read(&mmu->refcnt))) 796 kvm_free_stage2_pgd(mmu); 797 } 798 kvfree(kvm->arch.nested_mmus); 799 kvm->arch.nested_mmus = NULL; 800 kvm->arch.nested_mmus_size = 0; 801 kvm_uninit_stage2_mmu(kvm); 802 } 803 804 /* 805 * Our emulated CPU doesn't support all the possible features. For the 806 * sake of simplicity (and probably mental sanity), wipe out a number 807 * of feature bits we don't intend to support for the time being. 808 * This list should get updated as new features get added to the NV 809 * support, and new extension to the architecture. 810 */ 811 u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) 812 { 813 switch (reg) { 814 case SYS_ID_AA64ISAR0_EL1: 815 /* Support everything but TME */ 816 val &= ~ID_AA64ISAR0_EL1_TME; 817 break; 818 819 case SYS_ID_AA64ISAR1_EL1: 820 /* Support everything but LS64 and Spec Invalidation */ 821 val &= ~(ID_AA64ISAR1_EL1_LS64 | 822 ID_AA64ISAR1_EL1_SPECRES); 823 break; 824 825 case SYS_ID_AA64PFR0_EL1: 826 /* No RME, AMU, MPAM, S-EL2, or RAS */ 827 val &= ~(ID_AA64PFR0_EL1_RME | 828 ID_AA64PFR0_EL1_AMU | 829 ID_AA64PFR0_EL1_MPAM | 830 ID_AA64PFR0_EL1_SEL2 | 831 ID_AA64PFR0_EL1_RAS | 832 ID_AA64PFR0_EL1_EL3 | 833 ID_AA64PFR0_EL1_EL2 | 834 ID_AA64PFR0_EL1_EL1 | 835 ID_AA64PFR0_EL1_EL0); 836 /* 64bit only at any EL */ 837 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL0, IMP); 838 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL1, IMP); 839 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL2, IMP); 840 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, EL3, IMP); 841 break; 842 843 case SYS_ID_AA64PFR1_EL1: 844 /* Only support BTI, SSBS, CSV2_frac */ 845 val &= (ID_AA64PFR1_EL1_BT | 846 ID_AA64PFR1_EL1_SSBS | 847 ID_AA64PFR1_EL1_CSV2_frac); 848 break; 849 850 case SYS_ID_AA64MMFR0_EL1: 851 /* Hide ExS, Secure Memory */ 852 val &= ~(ID_AA64MMFR0_EL1_EXS | 853 ID_AA64MMFR0_EL1_TGRAN4_2 | 854 ID_AA64MMFR0_EL1_TGRAN16_2 | 855 ID_AA64MMFR0_EL1_TGRAN64_2 | 856 ID_AA64MMFR0_EL1_SNSMEM); 857 858 /* Hide CNTPOFF if present */ 859 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, ECV, IMP); 860 861 /* Disallow unsupported S2 page sizes */ 862 switch (PAGE_SIZE) { 863 case SZ_64K: 864 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, NI); 865 fallthrough; 866 case SZ_16K: 867 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, NI); 868 fallthrough; 869 case SZ_4K: 870 /* Support everything */ 871 break; 872 } 873 874 /* 875 * Since we can't support a guest S2 page size smaller 876 * than the host's own page size (due to KVM only 877 * populating its own S2 using the kernel's page 878 * size), advertise the limitation using FEAT_GTG. 879 */ 880 switch (PAGE_SIZE) { 881 case SZ_4K: 882 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN4_2, IMP); 883 fallthrough; 884 case SZ_16K: 885 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN16_2, IMP); 886 fallthrough; 887 case SZ_64K: 888 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR0_EL1, TGRAN64_2, IMP); 889 break; 890 } 891 892 /* Cap PARange to 48bits */ 893 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR0_EL1, PARANGE, 48); 894 break; 895 896 case SYS_ID_AA64MMFR1_EL1: 897 val &= (ID_AA64MMFR1_EL1_HCX | 898 ID_AA64MMFR1_EL1_PAN | 899 ID_AA64MMFR1_EL1_LO | 900 ID_AA64MMFR1_EL1_HPDS | 901 ID_AA64MMFR1_EL1_VH | 902 ID_AA64MMFR1_EL1_VMIDBits); 903 /* FEAT_E2H0 implies no VHE */ 904 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) 905 val &= ~ID_AA64MMFR1_EL1_VH; 906 break; 907 908 case SYS_ID_AA64MMFR2_EL1: 909 val &= ~(ID_AA64MMFR2_EL1_BBM | 910 ID_AA64MMFR2_EL1_TTL | 911 GENMASK_ULL(47, 44) | 912 ID_AA64MMFR2_EL1_ST | 913 ID_AA64MMFR2_EL1_CCIDX | 914 ID_AA64MMFR2_EL1_VARange); 915 916 /* Force TTL support */ 917 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR2_EL1, TTL, IMP); 918 break; 919 920 case SYS_ID_AA64MMFR4_EL1: 921 /* 922 * You get EITHER 923 * 924 * - FEAT_VHE without FEAT_E2H0 925 * - FEAT_NV limited to FEAT_NV2 926 * - HCR_EL2.NV1 being RES0 927 * 928 * OR 929 * 930 * - FEAT_E2H0 without FEAT_VHE nor FEAT_NV 931 * 932 * Life is too short for anything else. 933 */ 934 if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) { 935 val = 0; 936 } else { 937 val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); 938 val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1); 939 } 940 break; 941 942 case SYS_ID_AA64DFR0_EL1: 943 /* Only limited support for PMU, Debug, BPs, WPs, and HPMN0 */ 944 val &= (ID_AA64DFR0_EL1_PMUVer | 945 ID_AA64DFR0_EL1_WRPs | 946 ID_AA64DFR0_EL1_BRPs | 947 ID_AA64DFR0_EL1_DebugVer| 948 ID_AA64DFR0_EL1_HPMN0); 949 950 /* Cap Debug to ARMv8.1 */ 951 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, VHE); 952 break; 953 } 954 955 return val; 956 } 957 958 u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *vcpu, 959 enum vcpu_sysreg sr, u64 v) 960 { 961 struct kvm_sysreg_masks *masks; 962 963 masks = vcpu->kvm->arch.sysreg_masks; 964 965 if (masks) { 966 sr -= __SANITISED_REG_START__; 967 968 v &= ~masks->mask[sr].res0; 969 v |= masks->mask[sr].res1; 970 } 971 972 return v; 973 } 974 975 static __always_inline void set_sysreg_masks(struct kvm *kvm, int sr, u64 res0, u64 res1) 976 { 977 int i = sr - __SANITISED_REG_START__; 978 979 BUILD_BUG_ON(!__builtin_constant_p(sr)); 980 BUILD_BUG_ON(sr < __SANITISED_REG_START__); 981 BUILD_BUG_ON(sr >= NR_SYS_REGS); 982 983 kvm->arch.sysreg_masks->mask[i].res0 = res0; 984 kvm->arch.sysreg_masks->mask[i].res1 = res1; 985 } 986 987 int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) 988 { 989 struct kvm *kvm = vcpu->kvm; 990 u64 res0, res1; 991 992 lockdep_assert_held(&kvm->arch.config_lock); 993 994 if (kvm->arch.sysreg_masks) 995 goto out; 996 997 kvm->arch.sysreg_masks = kzalloc(sizeof(*(kvm->arch.sysreg_masks)), 998 GFP_KERNEL_ACCOUNT); 999 if (!kvm->arch.sysreg_masks) 1000 return -ENOMEM; 1001 1002 /* VTTBR_EL2 */ 1003 res0 = res1 = 0; 1004 if (!kvm_has_feat_enum(kvm, ID_AA64MMFR1_EL1, VMIDBits, 16)) 1005 res0 |= GENMASK(63, 56); 1006 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, CnP, IMP)) 1007 res0 |= VTTBR_CNP_BIT; 1008 set_sysreg_masks(kvm, VTTBR_EL2, res0, res1); 1009 1010 /* VTCR_EL2 */ 1011 res0 = GENMASK(63, 32) | GENMASK(30, 20); 1012 res1 = BIT(31); 1013 set_sysreg_masks(kvm, VTCR_EL2, res0, res1); 1014 1015 /* VMPIDR_EL2 */ 1016 res0 = GENMASK(63, 40) | GENMASK(30, 24); 1017 res1 = BIT(31); 1018 set_sysreg_masks(kvm, VMPIDR_EL2, res0, res1); 1019 1020 /* HCR_EL2 */ 1021 res0 = BIT(48); 1022 res1 = HCR_RW; 1023 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, TWED, IMP)) 1024 res0 |= GENMASK(63, 59); 1025 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, MTE, MTE2)) 1026 res0 |= (HCR_TID5 | HCR_DCT | HCR_ATA); 1027 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, TTLBxS)) 1028 res0 |= (HCR_TTLBIS | HCR_TTLBOS); 1029 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1030 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1031 res0 |= HCR_ENSCXT; 1032 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, EVT, IMP)) 1033 res0 |= (HCR_TOCU | HCR_TICAB | HCR_TID4); 1034 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1035 res0 |= HCR_AMVOFFEN; 1036 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, V1P1)) 1037 res0 |= HCR_FIEN; 1038 if (!kvm_has_feat(kvm, ID_AA64MMFR2_EL1, FWB, IMP)) 1039 res0 |= HCR_FWB; 1040 /* Implementation choice: NV2 is the only supported config */ 1041 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) 1042 res0 |= (HCR_NV2 | HCR_NV | HCR_AT); 1043 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, NI)) 1044 res0 |= HCR_NV1; 1045 if (!(kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1046 kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1047 res0 |= (HCR_API | HCR_APK); 1048 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TME, IMP)) 1049 res0 |= BIT(39); 1050 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1051 res0 |= (HCR_TEA | HCR_TERR); 1052 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1053 res0 |= HCR_TLOR; 1054 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1055 res0 |= HCR_E2H; 1056 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, E2H0, IMP)) 1057 res1 |= HCR_E2H; 1058 set_sysreg_masks(kvm, HCR_EL2, res0, res1); 1059 1060 /* HCRX_EL2 */ 1061 res0 = HCRX_EL2_RES0; 1062 res1 = HCRX_EL2_RES1; 1063 if (!kvm_has_feat(kvm, ID_AA64ISAR3_EL1, PACM, TRIVIAL_IMP)) 1064 res0 |= HCRX_EL2_PACMEn; 1065 if (!kvm_has_feat(kvm, ID_AA64PFR2_EL1, FPMR, IMP)) 1066 res0 |= HCRX_EL2_EnFPM; 1067 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1068 res0 |= HCRX_EL2_GCSEn; 1069 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, SYSREG_128, IMP)) 1070 res0 |= HCRX_EL2_EnIDCP128; 1071 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, ADERR, DEV_ASYNC)) 1072 res0 |= (HCRX_EL2_EnSDERR | HCRX_EL2_EnSNERR); 1073 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, DF2, IMP)) 1074 res0 |= HCRX_EL2_TMEA; 1075 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP)) 1076 res0 |= HCRX_EL2_D128En; 1077 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1078 res0 |= HCRX_EL2_PTTWI; 1079 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, SCTLRX, IMP)) 1080 res0 |= HCRX_EL2_SCTLR2En; 1081 if (!kvm_has_tcr2(kvm)) 1082 res0 |= HCRX_EL2_TCR2En; 1083 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP)) 1084 res0 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2); 1085 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, CMOW, IMP)) 1086 res0 |= HCRX_EL2_CMOW; 1087 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, NMI, IMP)) 1088 res0 |= (HCRX_EL2_VFNMI | HCRX_EL2_VINMI | HCRX_EL2_TALLINT); 1089 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP) || 1090 !(read_sysreg_s(SYS_SMIDR_EL1) & SMIDR_EL1_SMPS)) 1091 res0 |= HCRX_EL2_SMPME; 1092 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP)) 1093 res0 |= (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS); 1094 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) 1095 res0 |= HCRX_EL2_EnASR; 1096 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64)) 1097 res0 |= HCRX_EL2_EnALS; 1098 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1099 res0 |= HCRX_EL2_EnAS0; 1100 set_sysreg_masks(kvm, HCRX_EL2, res0, res1); 1101 1102 /* HFG[RW]TR_EL2 */ 1103 res0 = res1 = 0; 1104 if (!(kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_ADDRESS) && 1105 kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_PTRAUTH_GENERIC))) 1106 res0 |= (HFGxTR_EL2_APDAKey | HFGxTR_EL2_APDBKey | 1107 HFGxTR_EL2_APGAKey | HFGxTR_EL2_APIAKey | 1108 HFGxTR_EL2_APIBKey); 1109 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, LO, IMP)) 1110 res0 |= (HFGxTR_EL2_LORC_EL1 | HFGxTR_EL2_LOREA_EL1 | 1111 HFGxTR_EL2_LORID_EL1 | HFGxTR_EL2_LORN_EL1 | 1112 HFGxTR_EL2_LORSA_EL1); 1113 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, CSV2, CSV2_2) && 1114 !kvm_has_feat(kvm, ID_AA64PFR1_EL1, CSV2_frac, CSV2_1p2)) 1115 res0 |= (HFGxTR_EL2_SCXTNUM_EL1 | HFGxTR_EL2_SCXTNUM_EL0); 1116 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP)) 1117 res0 |= HFGxTR_EL2_ICC_IGRPENn_EL1; 1118 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) 1119 res0 |= (HFGxTR_EL2_ERRIDR_EL1 | HFGxTR_EL2_ERRSELR_EL1 | 1120 HFGxTR_EL2_ERXFR_EL1 | HFGxTR_EL2_ERXCTLR_EL1 | 1121 HFGxTR_EL2_ERXSTATUS_EL1 | HFGxTR_EL2_ERXMISCn_EL1 | 1122 HFGxTR_EL2_ERXPFGF_EL1 | HFGxTR_EL2_ERXPFGCTL_EL1 | 1123 HFGxTR_EL2_ERXPFGCDN_EL1 | HFGxTR_EL2_ERXADDR_EL1); 1124 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA)) 1125 res0 |= HFGxTR_EL2_nACCDATA_EL1; 1126 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1127 res0 |= (HFGxTR_EL2_nGCS_EL0 | HFGxTR_EL2_nGCS_EL1); 1128 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, SME, IMP)) 1129 res0 |= (HFGxTR_EL2_nSMPRI_EL1 | HFGxTR_EL2_nTPIDR2_EL0); 1130 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1131 res0 |= HFGxTR_EL2_nRCWMASK_EL1; 1132 if (!kvm_has_s1pie(kvm)) 1133 res0 |= (HFGxTR_EL2_nPIRE0_EL1 | HFGxTR_EL2_nPIR_EL1); 1134 if (!kvm_has_s1poe(kvm)) 1135 res0 |= (HFGxTR_EL2_nPOR_EL0 | HFGxTR_EL2_nPOR_EL1); 1136 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S2POE, IMP)) 1137 res0 |= HFGxTR_EL2_nS2POR_EL1; 1138 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP)) 1139 res0 |= (HFGxTR_EL2_nMAIR2_EL1 | HFGxTR_EL2_nAMAIR2_EL1); 1140 set_sysreg_masks(kvm, HFGRTR_EL2, res0 | __HFGRTR_EL2_RES0, res1); 1141 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | __HFGWTR_EL2_RES0, res1); 1142 1143 /* HDFG[RW]TR_EL2 */ 1144 res0 = res1 = 0; 1145 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DoubleLock, IMP)) 1146 res0 |= HDFGRTR_EL2_OSDLR_EL1; 1147 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1148 res0 |= (HDFGRTR_EL2_PMEVCNTRn_EL0 | HDFGRTR_EL2_PMEVTYPERn_EL0 | 1149 HDFGRTR_EL2_PMCCFILTR_EL0 | HDFGRTR_EL2_PMCCNTR_EL0 | 1150 HDFGRTR_EL2_PMCNTEN | HDFGRTR_EL2_PMINTEN | 1151 HDFGRTR_EL2_PMOVS | HDFGRTR_EL2_PMSELR_EL0 | 1152 HDFGRTR_EL2_PMMIR_EL1 | HDFGRTR_EL2_PMUSERENR_EL0 | 1153 HDFGRTR_EL2_PMCEIDn_EL0); 1154 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) 1155 res0 |= (HDFGRTR_EL2_PMBLIMITR_EL1 | HDFGRTR_EL2_PMBPTR_EL1 | 1156 HDFGRTR_EL2_PMBSR_EL1 | HDFGRTR_EL2_PMSCR_EL1 | 1157 HDFGRTR_EL2_PMSEVFR_EL1 | HDFGRTR_EL2_PMSFCR_EL1 | 1158 HDFGRTR_EL2_PMSICR_EL1 | HDFGRTR_EL2_PMSIDR_EL1 | 1159 HDFGRTR_EL2_PMSIRR_EL1 | HDFGRTR_EL2_PMSLATFR_EL1 | 1160 HDFGRTR_EL2_PMBIDR_EL1); 1161 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1162 res0 |= (HDFGRTR_EL2_TRC | HDFGRTR_EL2_TRCAUTHSTATUS | 1163 HDFGRTR_EL2_TRCAUXCTLR | HDFGRTR_EL2_TRCCLAIM | 1164 HDFGRTR_EL2_TRCCNTVRn | HDFGRTR_EL2_TRCID | 1165 HDFGRTR_EL2_TRCIMSPECn | HDFGRTR_EL2_TRCOSLSR | 1166 HDFGRTR_EL2_TRCPRGCTLR | HDFGRTR_EL2_TRCSEQSTR | 1167 HDFGRTR_EL2_TRCSSCSRn | HDFGRTR_EL2_TRCSTATR | 1168 HDFGRTR_EL2_TRCVICTLR); 1169 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 1170 res0 |= (HDFGRTR_EL2_TRBBASER_EL1 | HDFGRTR_EL2_TRBIDR_EL1 | 1171 HDFGRTR_EL2_TRBLIMITR_EL1 | HDFGRTR_EL2_TRBMAR_EL1 | 1172 HDFGRTR_EL2_TRBPTR_EL1 | HDFGRTR_EL2_TRBSR_EL1 | 1173 HDFGRTR_EL2_TRBTRG_EL1); 1174 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1175 res0 |= (HDFGRTR_EL2_nBRBIDR | HDFGRTR_EL2_nBRBCTL | 1176 HDFGRTR_EL2_nBRBDATA); 1177 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2)) 1178 res0 |= HDFGRTR_EL2_nPMSNEVFR_EL1; 1179 set_sysreg_masks(kvm, HDFGRTR_EL2, res0 | HDFGRTR_EL2_RES0, res1); 1180 1181 /* Reuse the bits from the read-side and add the write-specific stuff */ 1182 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1183 res0 |= (HDFGWTR_EL2_PMCR_EL0 | HDFGWTR_EL2_PMSWINC_EL0); 1184 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceVer, IMP)) 1185 res0 |= HDFGWTR_EL2_TRCOSLAR; 1186 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 1187 res0 |= HDFGWTR_EL2_TRFCR_EL1; 1188 set_sysreg_masks(kvm, HFGWTR_EL2, res0 | HDFGWTR_EL2_RES0, res1); 1189 1190 /* HFGITR_EL2 */ 1191 res0 = HFGITR_EL2_RES0; 1192 res1 = HFGITR_EL2_RES1; 1193 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, DPB, DPB2)) 1194 res0 |= HFGITR_EL2_DCCVADP; 1195 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN2)) 1196 res0 |= (HFGITR_EL2_ATS1E1RP | HFGITR_EL2_ATS1E1WP); 1197 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) 1198 res0 |= (HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1199 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS | 1200 HFGITR_EL2_TLBIVAALE1OS | HFGITR_EL2_TLBIVALE1OS | 1201 HFGITR_EL2_TLBIVAAE1OS | HFGITR_EL2_TLBIASIDE1OS | 1202 HFGITR_EL2_TLBIVAE1OS | HFGITR_EL2_TLBIVMALLE1OS); 1203 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE)) 1204 res0 |= (HFGITR_EL2_TLBIRVAALE1 | HFGITR_EL2_TLBIRVALE1 | 1205 HFGITR_EL2_TLBIRVAAE1 | HFGITR_EL2_TLBIRVAE1 | 1206 HFGITR_EL2_TLBIRVAALE1IS | HFGITR_EL2_TLBIRVALE1IS | 1207 HFGITR_EL2_TLBIRVAAE1IS | HFGITR_EL2_TLBIRVAE1IS | 1208 HFGITR_EL2_TLBIRVAALE1OS | HFGITR_EL2_TLBIRVALE1OS | 1209 HFGITR_EL2_TLBIRVAAE1OS | HFGITR_EL2_TLBIRVAE1OS); 1210 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, IMP)) 1211 res0 |= (HFGITR_EL2_CFPRCTX | HFGITR_EL2_DVPRCTX | 1212 HFGITR_EL2_CPPRCTX); 1213 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, BRBE, IMP)) 1214 res0 |= (HFGITR_EL2_nBRBINJ | HFGITR_EL2_nBRBIALL); 1215 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, GCS, IMP)) 1216 res0 |= (HFGITR_EL2_nGCSPUSHM_EL1 | HFGITR_EL2_nGCSSTR_EL1 | 1217 HFGITR_EL2_nGCSEPP); 1218 if (!kvm_has_feat(kvm, ID_AA64ISAR1_EL1, SPECRES, COSP_RCTX)) 1219 res0 |= HFGITR_EL2_COSPRCTX; 1220 if (!kvm_has_feat(kvm, ID_AA64ISAR2_EL1, ATS1A, IMP)) 1221 res0 |= HFGITR_EL2_ATS1E1A; 1222 set_sysreg_masks(kvm, HFGITR_EL2, res0, res1); 1223 1224 /* HAFGRTR_EL2 - not a lot to see here */ 1225 res0 = HAFGRTR_EL2_RES0; 1226 res1 = HAFGRTR_EL2_RES1; 1227 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, V1P1)) 1228 res0 |= ~(res0 | res1); 1229 set_sysreg_masks(kvm, HAFGRTR_EL2, res0, res1); 1230 1231 /* TCR2_EL2 */ 1232 res0 = TCR2_EL2_RES0; 1233 res1 = TCR2_EL2_RES1; 1234 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP)) 1235 res0 |= (TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1 | TCR2_EL2_D128); 1236 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, MEC, IMP)) 1237 res0 |= TCR2_EL2_AMEC1 | TCR2_EL2_AMEC0; 1238 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, HAFDBS, HAFT)) 1239 res0 |= TCR2_EL2_HAFT; 1240 if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP)) 1241 res0 |= TCR2_EL2_PTTWI | TCR2_EL2_PnCH; 1242 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP)) 1243 res0 |= TCR2_EL2_AIE; 1244 if (!kvm_has_s1poe(kvm)) 1245 res0 |= TCR2_EL2_POE | TCR2_EL2_E0POE; 1246 if (!kvm_has_s1pie(kvm)) 1247 res0 |= TCR2_EL2_PIE; 1248 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1249 res0 |= (TCR2_EL2_E0POE | TCR2_EL2_D128 | 1250 TCR2_EL2_AMEC1 | TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1); 1251 set_sysreg_masks(kvm, TCR2_EL2, res0, res1); 1252 1253 /* SCTLR_EL1 */ 1254 res0 = SCTLR_EL1_RES0; 1255 res1 = SCTLR_EL1_RES1; 1256 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3)) 1257 res0 |= SCTLR_EL1_EPAN; 1258 set_sysreg_masks(kvm, SCTLR_EL1, res0, res1); 1259 1260 /* MDCR_EL2 */ 1261 res0 = MDCR_EL2_RES0; 1262 res1 = MDCR_EL2_RES1; 1263 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) 1264 res0 |= (MDCR_EL2_HPMN | MDCR_EL2_TPMCR | 1265 MDCR_EL2_TPM | MDCR_EL2_HPME); 1266 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP)) 1267 res0 |= MDCR_EL2_E2PB | MDCR_EL2_TPMS; 1268 if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, SPMU, IMP)) 1269 res0 |= MDCR_EL2_EnSPM; 1270 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P1)) 1271 res0 |= MDCR_EL2_HPMD; 1272 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP)) 1273 res0 |= MDCR_EL2_TTRF; 1274 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P5)) 1275 res0 |= MDCR_EL2_HCCD | MDCR_EL2_HLP; 1276 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP)) 1277 res0 |= MDCR_EL2_E2TB; 1278 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP)) 1279 res0 |= MDCR_EL2_TDCC; 1280 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, MTPMU, IMP) || 1281 kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP)) 1282 res0 |= MDCR_EL2_MTPME; 1283 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P7)) 1284 res0 |= MDCR_EL2_HPMFZO; 1285 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSS, IMP)) 1286 res0 |= MDCR_EL2_PMSSE; 1287 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2)) 1288 res0 |= MDCR_EL2_HPMFZS; 1289 if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, EBEP, IMP)) 1290 res0 |= MDCR_EL2_PMEE; 1291 if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, V8P9)) 1292 res0 |= MDCR_EL2_EBWE; 1293 if (!kvm_has_feat(kvm, ID_AA64DFR2_EL1, STEP, IMP)) 1294 res0 |= MDCR_EL2_EnSTEPOP; 1295 set_sysreg_masks(kvm, MDCR_EL2, res0, res1); 1296 1297 /* CNTHCTL_EL2 */ 1298 res0 = GENMASK(63, 20); 1299 res1 = 0; 1300 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RME, IMP)) 1301 res0 |= CNTHCTL_CNTPMASK | CNTHCTL_CNTVMASK; 1302 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, CNTPOFF)) { 1303 res0 |= CNTHCTL_ECV; 1304 if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, ECV, IMP)) 1305 res0 |= (CNTHCTL_EL1TVT | CNTHCTL_EL1TVCT | 1306 CNTHCTL_EL1NVPCT | CNTHCTL_EL1NVVCT); 1307 } 1308 if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP)) 1309 res0 |= GENMASK(11, 8); 1310 set_sysreg_masks(kvm, CNTHCTL_EL2, res0, res1); 1311 1312 /* ICH_HCR_EL2 */ 1313 res0 = ICH_HCR_EL2_RES0; 1314 res1 = ICH_HCR_EL2_RES1; 1315 if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS)) 1316 res0 |= ICH_HCR_EL2_TDIR; 1317 /* No GICv4 is presented to the guest */ 1318 res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; 1319 set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1); 1320 1321 out: 1322 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) 1323 (void)__vcpu_sys_reg(vcpu, sr); 1324 1325 return 0; 1326 } 1327 1328 void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) 1329 { 1330 if (kvm_check_request(KVM_REQ_NESTED_S2_UNMAP, vcpu)) { 1331 struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; 1332 1333 write_lock(&vcpu->kvm->mmu_lock); 1334 if (mmu->pending_unmap) { 1335 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), true); 1336 mmu->pending_unmap = false; 1337 } 1338 write_unlock(&vcpu->kvm->mmu_lock); 1339 } 1340 1341 /* Must be last, as may switch context! */ 1342 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) 1343 kvm_inject_nested_irq(vcpu); 1344 } 1345