Lines Matching +full:page +full:- +full:level
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
9 #include <linux/page-flags.h>
18 return slot->arch.flags & KVM_MEM_HUGEPAGE_CAPABLE; in kvm_hugepage_capable()
23 return slot->arch.flags & KVM_MEM_HUGEPAGE_INCAPABLE; in kvm_hugepage_incapable()
28 ctx->level = kvm->arch.root_level; in kvm_ptw_prepare()
30 ctx->invalid_ptes = kvm->arch.invalid_ptes; in kvm_ptw_prepare()
31 ctx->pte_shifts = kvm->arch.pte_shifts; in kvm_ptw_prepare()
32 ctx->pgtable_shift = ctx->pte_shifts[ctx->level]; in kvm_ptw_prepare()
33 ctx->invalid_entry = ctx->invalid_ptes[ctx->level]; in kvm_ptw_prepare()
34 ctx->opaque = kvm; in kvm_ptw_prepare()
39 * VM's GPA page table to allow detection of commonly used pages.
53 * GPA page table to allow dirty page tracking.
63 * may cross hugepage, for first huge page parameter addr is equal to in kvm_mkclean_pte()
64 * start, however for the second huge page addr is base address of in kvm_mkclean_pte()
65 * this huge page, rather than start or end address in kvm_mkclean_pte()
67 if ((ctx->flag & _KVM_HAS_PGMASK) && !kvm_pte_huge(val)) { in kvm_mkclean_pte()
68 offset = (addr >> PAGE_SHIFT) - ctx->gfn; in kvm_mkclean_pte()
69 if (!(BIT(offset) & ctx->mask)) in kvm_mkclean_pte()
74 * Need not split huge page now, just set write-proect pte bit in kvm_mkclean_pte()
75 * Split huge page until next write fault in kvm_mkclean_pte()
92 kvm = ctx->opaque; in kvm_flush_pte()
93 if (ctx->level) in kvm_flush_pte()
94 kvm->stat.hugepages--; in kvm_flush_pte()
96 kvm->stat.pages--; in kvm_flush_pte()
98 *pte = ctx->invalid_entry; in kvm_flush_pte()
104 * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory.
106 * Allocate a blank KVM GPA page directory (PGD) for representing guest physical
107 * to host physical page mappings.
109 * Returns: Pointer to new KVM GPA page directory.
136 p[-3] = val; in _kvm_pte_init()
137 p[-2] = val; in _kvm_pte_init()
138 p[-1] = val; in _kvm_pte_init()
143 * Caller must hold kvm->mm_lock
145 * Walk the page tables of kvm to find the PTE corresponding to the
146 * address @addr. If page tables don't exist for @addr, they will be created
151 unsigned long addr, int level) in kvm_populate_gpa() argument
157 child = kvm->arch.pgd; in kvm_populate_gpa()
158 while (ctx.level > level) { in kvm_populate_gpa()
165 _kvm_pte_init(child, ctx.invalid_ptes[ctx.level - 1]); in kvm_populate_gpa()
180 * Page walker for VM shadow mmu at last level
181 * The last level is small pte page or huge pmd page
195 next = addr + (0x1UL << ctx->pgtable_shift); in kvm_ptw_leaf()
199 ret |= ctx->ops(entry, addr, ctx); in kvm_ptw_leaf()
203 size = 0x1UL << (ctx->pgtable_shift + PAGE_SHIFT - 3); in kvm_ptw_leaf()
206 list_add_tail(list, &ctx->list); in kvm_ptw_leaf()
207 *dir = ctx->invalid_ptes[ctx->level + 1]; in kvm_ptw_leaf()
215 * Page walker for VM shadow mmu at page table dir level
234 ret |= ctx->ops(entry, addr, ctx); in kvm_ptw_dir()
239 if (ctx->level == 0) in kvm_ptw_dir()
247 size = 0x1UL << (ctx->pgtable_shift + PAGE_SHIFT - 3); in kvm_ptw_dir()
250 list_add_tail(list, &ctx->list); in kvm_ptw_dir()
251 *dir = ctx->invalid_ptes[ctx->level + 1]; in kvm_ptw_dir()
259 * Page walker for VM shadow mmu at page root table
283 * kvm_flush_range() - Flush a range of guest physical addresses.
285 * @start_gfn: Guest frame number of first page in GPA range to flush.
286 * @end_gfn: Guest frame number of last page in GPA range to flush.
289 * Flushes a range of GPA mappings from the GPA page tables.
303 spin_lock(&kvm->mmu_lock); in kvm_flush_range()
304 ret = kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT, in kvm_flush_range()
306 spin_unlock(&kvm->mmu_lock); in kvm_flush_range()
308 ret = kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT, in kvm_flush_range()
316 * free pte table page after mmu_lock in kvm_flush_range()
317 * the pte table page is linked together with ctx.list in kvm_flush_range()
326 * kvm_mkclean_gpa_pt() - Make a range of guest physical addresses clean.
328 * @start_gfn: Guest frame number of first page in GPA range to flush.
329 * @end_gfn: Guest frame number of last page in GPA range to flush.
332 * trigger dirty page logging.
334 * The caller must hold the @kvm->mmu_lock spinlock.
337 * derived mappings (GVA page tables & TLB enties) to be
347 return kvm_ptw_top(kvm->arch.pgd, start_gfn << PAGE_SHIFT, end_gfn << PAGE_SHIFT, &ctx); in kvm_mkclean_gpa_pt()
351 * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages
359 * acquire @kvm->mmu_lock.
365 gfn_t base_gfn = slot->base_gfn + gfn_offset; in kvm_arch_mmu_enable_log_dirty_pt_masked()
375 kvm_ptw_top(kvm->arch.pgd, start << PAGE_SHIFT, end << PAGE_SHIFT, &ctx); in kvm_arch_mmu_enable_log_dirty_pt_masked()
391 if ((new->base_gfn + new->npages) > (kvm->arch.gpa_size >> PAGE_SHIFT)) in kvm_arch_prepare_memory_region()
392 return -ENOMEM; in kvm_arch_prepare_memory_region()
394 new->arch.flags = 0; in kvm_arch_prepare_memory_region()
395 size = new->npages * PAGE_SIZE; in kvm_arch_prepare_memory_region()
396 gpa_start = new->base_gfn << PAGE_SHIFT; in kvm_arch_prepare_memory_region()
397 hva_start = new->userspace_addr; in kvm_arch_prepare_memory_region()
400 new->arch.flags |= KVM_MEM_HUGEPAGE_CAPABLE; in kvm_arch_prepare_memory_region()
410 * memslot->userspace_addr: in kvm_arch_prepare_memory_region()
411 * +-----+--------------------+--------------------+---+ in kvm_arch_prepare_memory_region()
412 * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| in kvm_arch_prepare_memory_region()
413 * +-----+--------------------+--------------------+---+ in kvm_arch_prepare_memory_region()
415 * memslot->base_gfn << PAGE_SIZE: in kvm_arch_prepare_memory_region()
416 * +---+--------------------+--------------------+-----+ in kvm_arch_prepare_memory_region()
417 * |abc|def Stage-2 block | Stage-2 block |tvxyz| in kvm_arch_prepare_memory_region()
418 * +---+--------------------+--------------------+-----+ in kvm_arch_prepare_memory_region()
420 * If we create those stage-2 blocks, we'll end up with this in kvm_arch_prepare_memory_region()
422 * d -> f in kvm_arch_prepare_memory_region()
423 * e -> g in kvm_arch_prepare_memory_region()
424 * f -> h in kvm_arch_prepare_memory_region()
426 gpa_offset = gpa_start & (PMD_SIZE - 1); in kvm_arch_prepare_memory_region()
427 hva_offset = hva_start & (PMD_SIZE - 1); in kvm_arch_prepare_memory_region()
429 new->arch.flags |= KVM_MEM_HUGEPAGE_INCAPABLE; in kvm_arch_prepare_memory_region()
434 new->arch.flags |= KVM_MEM_HUGEPAGE_INCAPABLE; in kvm_arch_prepare_memory_region()
449 * If dirty page logging is enabled, write protect all pages in the slot in kvm_arch_commit_memory_region()
458 (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && in kvm_arch_commit_memory_region()
459 new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { in kvm_arch_commit_memory_region()
460 spin_lock(&kvm->mmu_lock); in kvm_arch_commit_memory_region()
461 /* Write protect GPA page table entries */ in kvm_arch_commit_memory_region()
462 needs_flush = kvm_mkclean_gpa_pt(kvm, new->base_gfn, in kvm_arch_commit_memory_region()
463 new->base_gfn + new->npages); in kvm_arch_commit_memory_region()
464 spin_unlock(&kvm->mmu_lock); in kvm_arch_commit_memory_region()
472 kvm_flush_range(kvm, 0, kvm->arch.gpa_size >> PAGE_SHIFT, 0); in kvm_arch_flush_shadow_all()
481 kvm_flush_range(kvm, slot->base_gfn, slot->base_gfn + slot->npages, 1); in kvm_arch_flush_shadow_memslot()
493 return kvm_ptw_top(kvm->arch.pgd, range->start << PAGE_SHIFT, in kvm_unmap_gfn_range()
494 range->end << PAGE_SHIFT, &ctx); in kvm_unmap_gfn_range()
501 kvm_pfn_t pfn = pte_pfn(range->arg.pte); in kvm_set_spte_gfn()
502 gpa_t gpa = range->start << PAGE_SHIFT; in kvm_set_spte_gfn()
508 /* Replacing an absent or old page doesn't need flushes */ in kvm_set_spte_gfn()
514 /* Fill new pte if write protected or page migrated */ in kvm_set_spte_gfn()
516 prot_bits |= _CACHE_MASK & pte_val(range->arg.pte); in kvm_set_spte_gfn()
520 * _PAGE_WRITE for map_page_fast if next page write fault in kvm_set_spte_gfn()
521 * _PAGE_DIRTY since gpa has already recorded as dirty page in kvm_set_spte_gfn()
523 prot_bits |= __WRITEABLE & *ptep & pte_val(range->arg.pte); in kvm_set_spte_gfn()
537 return kvm_ptw_top(kvm->arch.pgd, range->start << PAGE_SHIFT, in kvm_age_gfn()
538 range->end << PAGE_SHIFT, &ctx); in kvm_age_gfn()
543 gpa_t gpa = range->start << PAGE_SHIFT; in kvm_test_age_gfn()
553 * kvm_map_page_fast() - Fast path GPA fault handler.
559 * calling into KVM. This handles marking old pages young (for idle page
560 * tracking), and dirtying of clean pages (for dirty page logging).
564 * -EFAULT on failure due to absent GPA mapping or write to
565 * read-only page, in which case KVM must be consulted.
573 struct kvm *kvm = vcpu->kvm; in kvm_map_page_fast()
576 spin_lock(&kvm->mmu_lock); in kvm_map_page_fast()
578 /* Fast path - just check GPA page table for an existing entry */ in kvm_map_page_fast()
581 ret = -EFAULT; in kvm_map_page_fast()
593 ret = -EFAULT; in kvm_map_page_fast()
604 ret = -EFAULT; in kvm_map_page_fast()
618 spin_unlock(&kvm->mmu_lock); in kvm_map_page_fast()
633 spin_unlock(&kvm->mmu_lock); in kvm_map_page_fast()
652 start = memslot->userspace_addr; in fault_supports_huge_mapping()
653 end = start + memslot->npages * PAGE_SIZE; in fault_supports_huge_mapping()
658 * for the beginning and end of a non-block aligned and non-block sized in fault_supports_huge_mapping()
671 * Lookup the mapping level for @gfn in the current mm.
678 * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before
682 * - Hold mmu_lock AND ensure there is no in-progress MMU notifier invalidation
686 * - Do not use the result to install new mappings, e.g. use the host mapping
687 * level only to decide whether or not to zap an entry. In this case, it's
691 * Note! The lookup can still race with modifications to host page tables, but
698 int level = 0; in host_pfn_mapping_level() local
707 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot() in host_pfn_mapping_level()
710 * read-only memslots due to gfn_to_hva() assuming writes. Earlier in host_pfn_mapping_level()
711 * page fault steps have already verified the guest isn't writing a in host_pfn_mapping_level()
712 * read-only memslot. in host_pfn_mapping_level()
717 * Disable IRQs to prevent concurrent tear down of host page tables, in host_pfn_mapping_level()
718 * e.g. if the primary MMU promotes a P*D to a huge page and then frees in host_pfn_mapping_level()
719 * the original page table. in host_pfn_mapping_level()
724 * Read each entry once. As above, a non-leaf entry can be promoted to in host_pfn_mapping_level()
725 * a huge page _during_ this walk. Re-reading the entry could send the in host_pfn_mapping_level()
727 * value) and then p*d_offset() walks into the target huge page instead in host_pfn_mapping_level()
728 * of the old page table (sees the new value). in host_pfn_mapping_level()
730 pgd = READ_ONCE(*pgd_offset(kvm->mm, hva)); in host_pfn_mapping_level()
747 level = 1; in host_pfn_mapping_level()
751 return level; in host_pfn_mapping_level()
755 * Split huge page
761 struct kvm *kvm = vcpu->kvm; in kvm_split_huge()
764 memcache = &vcpu->arch.mmu_page_cache; in kvm_split_huge()
775 kvm->stat.hugepages--; in kvm_split_huge()
776 kvm->stat.pages += PTRS_PER_PTE; in kvm_split_huge()
778 return child + (gfn & (PTRS_PER_PTE - 1)); in kvm_split_huge()
782 * kvm_map_page() - Map a guest physical page.
790 * This takes care of marking pages young or dirty (idle/dirty page tracking),
791 * asking KVM for the corresponding PFN, and creating a mapping in the GPA page
792 * tables. Derived mappings (GVA page tables and TLBs) must be handled by the
796 * -EFAULT if there is no memory region at @gpa or a write was
797 * attempted to a read-only memory region. This is usually handled
803 int srcu_idx, err, retry_no = 0, level; in kvm_map_page() local
808 struct kvm *kvm = vcpu->kvm; in kvm_map_page()
810 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; in kvm_map_page()
813 srcu_idx = srcu_read_lock(&kvm->srcu); in kvm_map_page()
821 err = -EFAULT; in kvm_map_page()
825 /* We need a minimum of cached pages ready for page table creation */ in kvm_map_page()
835 mmu_seq = kvm->mmu_invalidate_seq; in kvm_map_page()
839 * risk the page we get a reference to getting unmapped before we have a in kvm_map_page()
844 * spin_lock() in kvm_mmu_invalidate_invalidate_<page|range_end>() before in kvm_map_page()
849 /* Slow path - ask KVM core whether we can access this GPA */ in kvm_map_page()
852 err = -EFAULT; in kvm_map_page()
857 spin_lock(&kvm->mmu_lock); in kvm_map_page()
864 spin_unlock(&kvm->mmu_lock); in kvm_map_page()
892 level = 0; in kvm_map_page()
894 level = 0; in kvm_map_page()
896 level = host_pfn_mapping_level(kvm, gfn, memslot); in kvm_map_page()
897 if (level == 1) { in kvm_map_page()
898 gfn = gfn & ~(PTRS_PER_PTE - 1); in kvm_map_page()
899 pfn = pfn & ~(PTRS_PER_PTE - 1); in kvm_map_page()
903 /* Ensure page tables are allocated */ in kvm_map_page()
904 ptep = kvm_populate_gpa(kvm, memcache, gpa, level); in kvm_map_page()
906 if (level == 1) { in kvm_map_page()
910 * there is invalid tlb with small page in kvm_map_page()
914 ++kvm->stat.hugepages; in kvm_map_page()
918 ++kvm->stat.pages; in kvm_map_page()
920 spin_unlock(&kvm->mmu_lock); in kvm_map_page()
930 srcu_read_unlock(&kvm->srcu, srcu_idx); in kvm_map_page()