56 static u64 kvm_granule_shift(u32 level)  in kvm_granule_shift()  argument
59 	return ARM64_HW_PGTABLE_LEVEL_SHIFT(level);  in kvm_granule_shift()
62 static u64 kvm_granule_size(u32 level)  in kvm_granule_size()  argument
64 	return BIT(kvm_granule_shift(level));  in kvm_granule_size()
67 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)  in kvm_block_mapping_supported()  argument
69 	u64 granule = kvm_granule_size(level);  in kvm_block_mapping_supported()
75 	if (level == 0 || (PAGE_SIZE != SZ_4K && level == 1))  in kvm_block_mapping_supported()
84 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level)  in kvm_pgtable_idx()  argument
86 	u64 shift = kvm_granule_shift(level);  in kvm_pgtable_idx()
120 static bool kvm_pte_table(kvm_pte_t pte, u32 level)  in kvm_pte_table()  argument
122 	if (level == KVM_PGTABLE_MAX_LEVELS - 1)  in kvm_pte_table()
174 				   u32 level)  in kvm_set_valid_leaf_pte()  argument
177 	u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE :  in kvm_set_valid_leaf_pte()
193 				  u32 level, kvm_pte_t *ptep,  in kvm_pgtable_visitor_cb()  argument
197 	return walker->cb(addr, data->end, level, ptep, flag, walker->arg);  in kvm_pgtable_visitor_cb()
201 			      kvm_pte_t *pgtable, u32 level);
204 				      kvm_pte_t *ptep, u32 level)  in __kvm_pgtable_visit()  argument
209 	bool table = kvm_pte_table(pte, level);  in __kvm_pgtable_visit()
213 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
218 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
221 		table = kvm_pte_table(pte, level);  in __kvm_pgtable_visit()
228 		data->addr += kvm_granule_size(level);  in __kvm_pgtable_visit()
233 	ret = __kvm_pgtable_walk(data, childp, level + 1);  in __kvm_pgtable_visit()
238 		ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,  in __kvm_pgtable_visit()
247 			      kvm_pte_t *pgtable, u32 level)  in __kvm_pgtable_walk()  argument
252 	if (WARN_ON_ONCE(level >= KVM_PGTABLE_MAX_LEVELS))  in __kvm_pgtable_walk()
255 	for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {  in __kvm_pgtable_walk()
261 		ret = __kvm_pgtable_visit(data, ptep, level);  in __kvm_pgtable_walk()
341 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,  in hyp_map_walker_try_leaf()  argument
344 	u64 granule = kvm_granule_size(level), phys = data->phys;  in hyp_map_walker_try_leaf()
346 	if (!kvm_block_mapping_supported(addr, end, phys, level))  in hyp_map_walker_try_leaf()
349 	WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level));  in hyp_map_walker_try_leaf()
354 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_map_walker()  argument
359 	if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))  in hyp_map_walker()
362 	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))  in hyp_map_walker()
410 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in hyp_free_walker()  argument
464 static bool stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,  in stage2_map_walker_try_leaf()  argument
468 	u64 granule = kvm_granule_size(level), phys = data->phys;  in stage2_map_walker_try_leaf()
470 	if (!kvm_block_mapping_supported(addr, end, phys, level))  in stage2_map_walker_try_leaf()
482 	if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level))  in stage2_map_walker_try_leaf()
487 	kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);  in stage2_map_walker_try_leaf()
488 	kvm_set_valid_leaf_pte(ptep, phys, data->attr, level);  in stage2_map_walker_try_leaf()
494 static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,  in stage2_map_walk_table_pre()  argument
501 	if (!kvm_block_mapping_supported(addr, end, data->phys, level))  in stage2_map_walk_table_pre()
516 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walk_leaf()  argument
529 	if (stage2_map_walker_try_leaf(addr, end, level, ptep, data))  in stage2_map_walk_leaf()
532 	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))  in stage2_map_walk_leaf()
549 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);  in stage2_map_walk_leaf()
560 static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level,  in stage2_map_walk_table_post()  argument
574 		ret = stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walk_table_post()
599 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_map_walker()  argument
606 		return stage2_map_walk_table_pre(addr, end, level, ptep, data);  in stage2_map_walker()
608 		return stage2_map_walk_leaf(addr, end, level, ptep, data);  in stage2_map_walker()
610 		return stage2_map_walk_table_post(addr, end, level, ptep, data);  in stage2_map_walker()
657 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_unmap_walker()  argument
668 	if (kvm_pte_table(pte, level)) {  in stage2_unmap_walker()
683 	kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level);  in stage2_unmap_walker()
688 				    kvm_granule_size(level));  in stage2_unmap_walker()
712 	u32		level;  member
715 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_attr_walker()  argument
725 	data->level = level;  in stage2_attr_walker()
744 				    u32 *level)  in stage2_update_leaf_attrs()  argument
765 	if (level)  in stage2_update_leaf_attrs()
766 		*level = data.level;  in stage2_update_leaf_attrs()
811 	u32 level;  in kvm_pgtable_stage2_relax_perms()  local
823 	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level);  in kvm_pgtable_stage2_relax_perms()
825 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);  in kvm_pgtable_stage2_relax_perms()
829 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_flush_walker()  argument
838 	stage2_flush_dcache(kvm_pte_follow(pte), kvm_granule_size(level));  in stage2_flush_walker()
877 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,  in stage2_free_walker()  argument
888 	if (kvm_pte_table(pte, level))  in stage2_free_walker()