191 				    struct io_pgtable_cfg *cfg,  in __arm_lpae_alloc_pages()  argument
194 	struct device *dev = cfg->iommu_dev;  in __arm_lpae_alloc_pages()
201 	if (cfg->alloc) {  in __arm_lpae_alloc_pages()
202 		pages = cfg->alloc(cookie, size, gfp);  in __arm_lpae_alloc_pages()
213 	if (!cfg->coherent_walk) {  in __arm_lpae_alloc_pages()
233 	if (cfg->free)  in __arm_lpae_alloc_pages()
234 		cfg->free(cookie, pages, size);  in __arm_lpae_alloc_pages()
242 				  struct io_pgtable_cfg *cfg,  in __arm_lpae_free_pages()  argument
245 	if (!cfg->coherent_walk)  in __arm_lpae_free_pages()
246 		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),  in __arm_lpae_free_pages()
249 	if (cfg->free)  in __arm_lpae_free_pages()
250 		cfg->free(cookie, pages, size);  in __arm_lpae_free_pages()
256 				struct io_pgtable_cfg *cfg)  in __arm_lpae_sync_pte()  argument
258 	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),  in __arm_lpae_sync_pte()
262 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)  in __arm_lpae_clear_pte()  argument
267 	if (!cfg->coherent_walk)  in __arm_lpae_clear_pte()
268 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_clear_pte()
281 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_init_pte()  local
293 	if (!cfg->coherent_walk)  in __arm_lpae_init_pte()
294 		__arm_lpae_sync_pte(ptep, num_entries, cfg);  in __arm_lpae_init_pte()
335 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_install_table()  local
338 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_install_table()
350 	if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))  in arm_lpae_install_table()
354 	__arm_lpae_sync_pte(ptep, 1, cfg);  in arm_lpae_install_table()
369 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()  local
394 		cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie);  in __arm_lpae_map()
400 			__arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie);  in __arm_lpae_map()
401 	} else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {  in __arm_lpae_map()
402 		__arm_lpae_sync_pte(ptep, 1, cfg);  in __arm_lpae_map()
473 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_prot_to_pte()
487 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_map_pages()  local
491 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_map_pages()
493 	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))  in arm_lpae_map_pages()
496 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)  in arm_lpae_map_pages()
498 	if (WARN_ON(iaext || paddr >> cfg->oas))  in arm_lpae_map_pages()
545 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie);  in __arm_lpae_free_pgtable()
562 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_split_blk_unmap()  local
573 	tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg, data->iop.cookie);  in arm_lpae_split_blk_unmap()
596 		__arm_lpae_free_pages(tablep, tablesz, cfg, data->iop.cookie);  in arm_lpae_split_blk_unmap()
645 			__arm_lpae_clear_pte(ptep, &iop->cfg);  in __arm_lpae_unmap()
680 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_unmap_pages()  local
682 	long iaext = (s64)iova >> cfg->ias;  in arm_lpae_unmap_pages()
684 	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))  in arm_lpae_unmap_pages()
687 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)  in arm_lpae_unmap_pages()
732 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)  in arm_lpae_restrict_pgsizes()  argument
744 	if (cfg->pgsize_bitmap & PAGE_SIZE)  in arm_lpae_restrict_pgsizes()
746 	else if (cfg->pgsize_bitmap & ~PAGE_MASK)  in arm_lpae_restrict_pgsizes()
747 		granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);  in arm_lpae_restrict_pgsizes()
748 	else if (cfg->pgsize_bitmap & PAGE_MASK)  in arm_lpae_restrict_pgsizes()
749 		granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);  in arm_lpae_restrict_pgsizes()
763 		if (cfg->oas > 48)  in arm_lpae_restrict_pgsizes()
770 	cfg->pgsize_bitmap &= page_sizes;  in arm_lpae_restrict_pgsizes()
771 	cfg->ias = min(cfg->ias, max_addr_bits);  in arm_lpae_restrict_pgsizes()
772 	cfg->oas = min(cfg->oas, max_addr_bits);  in arm_lpae_restrict_pgsizes()
776 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)  in arm_lpae_alloc_pgtable()  argument
781 	arm_lpae_restrict_pgsizes(cfg);  in arm_lpae_alloc_pgtable()
783 	if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))  in arm_lpae_alloc_pgtable()
786 	if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
789 	if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)  in arm_lpae_alloc_pgtable()
796 	pg_shift = __ffs(cfg->pgsize_bitmap);  in arm_lpae_alloc_pgtable()
799 	va_bits = cfg->ias - pg_shift;  in arm_lpae_alloc_pgtable()
816 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s1()  argument
820 	typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;  in arm_64_lpae_alloc_pgtable_s1()
823 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |  in arm_64_lpae_alloc_pgtable_s1()
828 	data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s1()
833 	if (cfg->coherent_walk) {  in arm_64_lpae_alloc_pgtable_s1()
837 		if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)  in arm_64_lpae_alloc_pgtable_s1()
842 		if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))  in arm_64_lpae_alloc_pgtable_s1()
848 	tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;  in arm_64_lpae_alloc_pgtable_s1()
861 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s1()
887 	tcr->tsz = 64ULL - cfg->ias;  in arm_64_lpae_alloc_pgtable_s1()
899 	cfg->arm_lpae_s1_cfg.mair = reg;  in arm_64_lpae_alloc_pgtable_s1()
903 					   GFP_KERNEL, cfg, cookie);  in arm_64_lpae_alloc_pgtable_s1()
911 	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s1()
920 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_64_lpae_alloc_pgtable_s2()  argument
924 	typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;  in arm_64_lpae_alloc_pgtable_s2()
927 	if (cfg->quirks)  in arm_64_lpae_alloc_pgtable_s2()
930 	data = arm_lpae_alloc_pgtable(cfg);  in arm_64_lpae_alloc_pgtable_s2()
949 	if (cfg->coherent_walk) {  in arm_64_lpae_alloc_pgtable_s2()
974 	switch (cfg->oas) {  in arm_64_lpae_alloc_pgtable_s2()
1000 	vtcr->tsz = 64ULL - cfg->ias;  in arm_64_lpae_alloc_pgtable_s2()
1005 					   GFP_KERNEL, cfg, cookie);  in arm_64_lpae_alloc_pgtable_s2()
1013 	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);  in arm_64_lpae_alloc_pgtable_s2()
1022 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s1()  argument
1024 	if (cfg->ias > 32 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s1()
1027 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s1()
1028 	return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s1()
1032 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)  in arm_32_lpae_alloc_pgtable_s2()  argument
1034 	if (cfg->ias > 40 || cfg->oas > 40)  in arm_32_lpae_alloc_pgtable_s2()
1037 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_32_lpae_alloc_pgtable_s2()
1038 	return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);  in arm_32_lpae_alloc_pgtable_s2()
1042 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)  in arm_mali_lpae_alloc_pgtable()  argument
1047 	if (cfg->quirks)  in arm_mali_lpae_alloc_pgtable()
1050 	if (cfg->ias > 48 || cfg->oas > 40)  in arm_mali_lpae_alloc_pgtable()
1053 	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);  in arm_mali_lpae_alloc_pgtable()
1055 	data = arm_lpae_alloc_pgtable(cfg);  in arm_mali_lpae_alloc_pgtable()
1071 	cfg->arm_mali_lpae_cfg.memattr =  in arm_mali_lpae_alloc_pgtable()
1080 					   cfg, cookie);  in arm_mali_lpae_alloc_pgtable()
1087 	cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |  in arm_mali_lpae_alloc_pgtable()
1090 	if (cfg->coherent_walk)  in arm_mali_lpae_alloc_pgtable()
1091 		cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;  in arm_mali_lpae_alloc_pgtable()
1162 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()  local
1164 	pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n",  in arm_lpae_dump_ops()
1165 		cfg->pgsize_bitmap, cfg->ias);  in arm_lpae_dump_ops()
1178 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)  in arm_lpae_run_tests()  argument
1193 		cfg_cookie = cfg;  in arm_lpae_run_tests()
1194 		ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);  in arm_lpae_run_tests()
1217 		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {  in arm_lpae_run_tests()
1239 		size = 1UL << __ffs(cfg->pgsize_bitmap);  in arm_lpae_run_tests()
1253 		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {  in arm_lpae_run_tests()
1294 	struct io_pgtable_cfg cfg = {  in arm_lpae_do_selftests()  local
1306 			cfg.pgsize_bitmap = pgsize[i];  in arm_lpae_do_selftests()
1307 			cfg.ias = ias[j];  in arm_lpae_do_selftests()
1310 			if (arm_lpae_run_tests(&cfg))  in arm_lpae_do_selftests()