23 static uint64_t page_align(struct kvm_vm *vm, uint64_t v)  in page_align()  argument
25 	return (v + vm->page_size) & ~(vm->page_size - 1);  in page_align()
28 static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pgd_index()  argument
30 	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;  in pgd_index()
31 	uint64_t mask = (1UL << (vm->va_bits - shift)) - 1;  in pgd_index()
36 static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pud_index()  argument
38 	unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift;  in pud_index()
39 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;  in pud_index()
41 	TEST_ASSERT(vm->pgtable_levels == 4,  in pud_index()
42 		"Mode %d does not have 4 page table levels", vm->mode);  in pud_index()
47 static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pmd_index()  argument
49 	unsigned int shift = (vm->page_shift - 3) + vm->page_shift;  in pmd_index()
50 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;  in pmd_index()
52 	TEST_ASSERT(vm->pgtable_levels >= 3,  in pmd_index()
53 		"Mode %d does not have >= 3 page table levels", vm->mode);  in pmd_index()
58 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva)  in pte_index()  argument
60 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;  in pte_index()
61 	return (gva >> vm->page_shift) & mask;  in pte_index()
64 static inline bool use_lpa2_pte_format(struct kvm_vm *vm)  in use_lpa2_pte_format()  argument
66 	return (vm->page_size == SZ_4K || vm->page_size == SZ_16K) &&  in use_lpa2_pte_format()
67 	    (vm->pa_bits > 48 || vm->va_bits > 48);  in use_lpa2_pte_format()
70 static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs)  in addr_pte()  argument
74 	if (use_lpa2_pte_format(vm)) {  in addr_pte()
75 		pte = pa & PTE_ADDR_MASK_LPA2(vm->page_shift);  in addr_pte()
79 		pte = pa & PTE_ADDR_MASK(vm->page_shift);  in addr_pte()
80 		if (vm->page_shift == 16)  in addr_pte()
88 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte)  in pte_addr()  argument
92 	if (use_lpa2_pte_format(vm)) {  in pte_addr()
93 		pa = pte & PTE_ADDR_MASK_LPA2(vm->page_shift);  in pte_addr()
96 		pa = pte & PTE_ADDR_MASK(vm->page_shift);  in pte_addr()
97 		if (vm->page_shift == 16)  in pte_addr()
104 static uint64_t ptrs_per_pgd(struct kvm_vm *vm)  in ptrs_per_pgd()  argument
106 	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;  in ptrs_per_pgd()
107 	return 1 << (vm->va_bits - shift);  in ptrs_per_pgd()
110 static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm)  in ptrs_per_pte()  argument
112 	return 1 << (vm->page_shift - 3);  in ptrs_per_pte()
115 void virt_arch_pgd_alloc(struct kvm_vm *vm)  in virt_arch_pgd_alloc()  argument
117 	size_t nr_pages = page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size;  in virt_arch_pgd_alloc()
119 	if (vm->pgd_created)  in virt_arch_pgd_alloc()
122 	vm->pgd = vm_phy_pages_alloc(vm, nr_pages,  in virt_arch_pgd_alloc()
124 				     vm->memslots[MEM_REGION_PT]);  in virt_arch_pgd_alloc()
125 	vm->pgd_created = true;  in virt_arch_pgd_alloc()
128 static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,  in _virt_pg_map()  argument
135 	TEST_ASSERT((vaddr % vm->page_size) == 0,  in _virt_pg_map()
137 		"  vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);  in _virt_pg_map()
138 	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,  in _virt_pg_map()
139 		(vaddr >> vm->page_shift)),  in _virt_pg_map()
141 	TEST_ASSERT((paddr % vm->page_size) == 0,  in _virt_pg_map()
143 		"  paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);  in _virt_pg_map()
144 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,  in _virt_pg_map()
146 		"  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",  in _virt_pg_map()
147 		paddr, vm->max_gfn, vm->page_size);  in _virt_pg_map()
149 	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8;  in _virt_pg_map()
151 		*ptep = addr_pte(vm, vm_alloc_page_table(vm),  in _virt_pg_map()
154 	switch (vm->pgtable_levels) {  in _virt_pg_map()
156 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8;  in _virt_pg_map()
158 			*ptep = addr_pte(vm, vm_alloc_page_table(vm),  in _virt_pg_map()
162 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8;  in _virt_pg_map()
164 			*ptep = addr_pte(vm, vm_alloc_page_table(vm),  in _virt_pg_map()
168 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8;  in _virt_pg_map()
175 	if (!use_lpa2_pte_format(vm))  in _virt_pg_map()
178 	*ptep = addr_pte(vm, paddr, pg_attr);  in _virt_pg_map()
181 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)  in virt_arch_pg_map()  argument
185 	_virt_pg_map(vm, vaddr, paddr, attr_idx);  in virt_arch_pg_map()
188 uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva)  in virt_get_pte_hva()  argument
192 	if (!vm->pgd_created)  in virt_get_pte_hva()
195 	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8;  in virt_get_pte_hva()
199 	switch (vm->pgtable_levels) {  in virt_get_pte_hva()
201 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8;  in virt_get_pte_hva()
206 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8;  in virt_get_pte_hva()
211 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8;  in virt_get_pte_hva()
222 	TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);  in virt_get_pte_hva()
226 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)  in addr_arch_gva2gpa()  argument
228 	uint64_t *ptep = virt_get_pte_hva(vm, gva);  in addr_arch_gva2gpa()
230 	return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));  in addr_arch_gva2gpa()
233 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)  in pte_dump()  argument
242 	for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {  in pte_dump()
243 		ptep = addr_gpa2hva(vm, pte);  in pte_dump()
247 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1);  in pte_dump()
252 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)  in virt_arch_dump()  argument
254 	int level = 4 - (vm->pgtable_levels - 1);  in virt_arch_dump()
257 	if (!vm->pgd_created)  in virt_arch_dump()
260 	for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pgd(vm) * 8; pgd += 8) {  in virt_arch_dump()
261 		ptep = addr_gpa2hva(vm, pgd);  in virt_arch_dump()
265 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level);  in virt_arch_dump()
272 	struct kvm_vm *vm = vcpu->vm;  in aarch64_vcpu_setup()  local
280 		vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred);  in aarch64_vcpu_setup()
296 	switch (vm->mode) {  in aarch64_vcpu_setup()
320 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);  in aarch64_vcpu_setup()
323 	ttbr0_el1 = vm->pgd & GENMASK(47, vm->page_shift);  in aarch64_vcpu_setup()
326 	switch (vm->mode) {  in aarch64_vcpu_setup()
331 		ttbr0_el1 |= FIELD_GET(GENMASK(51, 48), vm->pgd) << 2;  in aarch64_vcpu_setup()
350 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);  in aarch64_vcpu_setup()
356 	tcr_el1 |= TCR_T0SZ(vm->va_bits);  in aarch64_vcpu_setup()
357 	if (use_lpa2_pte_format(vm))  in aarch64_vcpu_setup()
383 static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,  in __aarch64_vcpu_add()  argument
388 	struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);  in __aarch64_vcpu_add()
390 	stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :  in __aarch64_vcpu_add()
391 					     vm->page_size;  in __aarch64_vcpu_add()
392 	stack_vaddr = __vm_vaddr_alloc(vm, stack_size,  in __aarch64_vcpu_add()
402 struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,  in aarch64_vcpu_add()  argument
405 	struct kvm_vcpu *vcpu = __aarch64_vcpu_add(vm, vcpu_id, init);  in aarch64_vcpu_add()
412 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)  in vm_arch_vcpu_add()  argument
414 	return __aarch64_vcpu_add(vm, vcpu_id, NULL);  in vm_arch_vcpu_add()
504 void vm_init_descriptor_tables(struct kvm_vm *vm)  in vm_init_descriptor_tables()  argument
506 	vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),  in vm_init_descriptor_tables()
507 					vm->page_size, MEM_REGION_DATA);  in vm_init_descriptor_tables()
509 	*(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;  in vm_init_descriptor_tables()
512 void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec,  in vm_install_sync_handler()  argument
515 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);  in vm_install_sync_handler()
523 void vm_install_exception_handler(struct kvm_vm *vm, int vector,  in vm_install_exception_handler()  argument
526 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);  in vm_install_exception_handler()
641 void vm_vaddr_populate_bitmap(struct kvm_vm *vm)  in vm_vaddr_populate_bitmap()  argument
647 	sparsebit_set_num(vm->vpages_valid, 0,  in vm_vaddr_populate_bitmap()
648 			  (1ULL << vm->va_bits) >> vm->page_shift);  in vm_vaddr_populate_bitmap()