1 // SPDX-License-Identifier: GPL-2.0
8 #include <linux/dma-map-ops.h>
9 #include <linux/dma-direct.h>
23 /* Size can be defined by the coherent_pool command line */
49 static void dma_atomic_pool_size_add(gfp_t gfp, size_t size)  in dma_atomic_pool_size_add()  argument
52 		pool_size_dma += size;  in dma_atomic_pool_size_add()
54 		pool_size_dma32 += size;  in dma_atomic_pool_size_add()
56 		pool_size_kernel += size;  in dma_atomic_pool_size_add()
61 	unsigned long size;  in cma_in_zone()  local
69 	size = cma_get_size(cma);  in cma_in_zone()
70 	if (!size)  in cma_in_zone()
74 	end = cma_get_base(cma) + size - 1;  in cma_in_zone()
86 	struct page *page = NULL;  in atomic_pool_expand()  local
88 	int ret = -ENOMEM;  in atomic_pool_expand()
90 	/* Cannot allocate larger than MAX_ORDER-1 */  in atomic_pool_expand()
91 	order = min(get_order(pool_size), MAX_ORDER-1);  in atomic_pool_expand()
96 			page = dma_alloc_from_contiguous(NULL, 1 << order,  in atomic_pool_expand()
98 		if (!page)  in atomic_pool_expand()
99 			page = alloc_pages(gfp, order);  in atomic_pool_expand()
100 	} while (!page && order-- > 0);  in atomic_pool_expand()
101 	if (!page)  in atomic_pool_expand()
104 	arch_dma_prep_coherent(page, pool_size);  in atomic_pool_expand()
107 	addr = dma_common_contiguous_remap(page, pool_size,  in atomic_pool_expand()
113 	addr = page_to_virt(page);  in atomic_pool_expand()
117 	 * shrink so no re-encryption occurs in dma_direct_free().  in atomic_pool_expand()
119 	ret = set_memory_decrypted((unsigned long)page_to_virt(page),  in atomic_pool_expand()
123 	ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),  in atomic_pool_expand()
132 	ret = set_memory_encrypted((unsigned long)page_to_virt(page),  in atomic_pool_expand()
143 	__free_pages(page, order);  in atomic_pool_expand()
196 	 * sizes to 128KB per 1GB of memory, min 128KB, max MAX_ORDER-1.  in dma_atomic_pool_init()
208 		ret = -ENOMEM;  in dma_atomic_pool_init()
213 			ret = -ENOMEM;  in dma_atomic_pool_init()
219 			ret = -ENOMEM;  in dma_atomic_pool_init()
243 static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,  in __dma_alloc_from_pool()  argument
250 	addr = gen_pool_alloc(pool, size);  in __dma_alloc_from_pool()
255 	if (phys_addr_ok && !phys_addr_ok(dev, phys, size)) {  in __dma_alloc_from_pool()
256 		gen_pool_free(pool, addr, size);  in __dma_alloc_from_pool()
264 	memset(*cpu_addr, 0, size);  in __dma_alloc_from_pool()
268 struct page *dma_alloc_from_pool(struct device *dev, size_t size,  in dma_alloc_from_pool()  argument
273 	struct page *page;  in dma_alloc_from_pool()  local
276 		page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,  in dma_alloc_from_pool()
278 		if (page)  in dma_alloc_from_pool()
279 			return page;  in dma_alloc_from_pool()
286 bool dma_free_from_pool(struct device *dev, void *start, size_t size)  in dma_free_from_pool()  argument
291 		if (!gen_pool_has_addr(pool, (unsigned long)start, size))  in dma_free_from_pool()
293 		gen_pool_free(pool, (unsigned long)start, size);  in dma_free_from_pool()