1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * Contiguous Memory Allocator
5  * Copyright (c) 2010-2011 by Samsung Electronics.
45 	return PFN_PHYS(cma->base_pfn);  in cma_get_base()
50 	return cma->count << PAGE_SHIFT;  in cma_get_size()
55 	return cma->name;  in cma_get_name()
61 	if (align_order <= cma->order_per_bit)  in cma_bitmap_aligned_mask()
63 	return (1UL << (align_order - cma->order_per_bit)) - 1;  in cma_bitmap_aligned_mask()
73 	return (cma->base_pfn & ((1UL << align_order) - 1))  in cma_bitmap_aligned_offset()
74 		>> cma->order_per_bit;  in cma_bitmap_aligned_offset()
80 	return ALIGN(pages, 1UL << cma->order_per_bit) >> cma->order_per_bit;  in cma_bitmap_pages_to_bits()
88 	bitmap_no = (pfn - cma->base_pfn) >> cma->order_per_bit;  in cma_clear_bitmap()
91 	mutex_lock(&cma->lock);  in cma_clear_bitmap()
92 	bitmap_clear(cma->bitmap, bitmap_no, bitmap_count);  in cma_clear_bitmap()
93 	mutex_unlock(&cma->lock);  in cma_clear_bitmap()
98 	unsigned long base_pfn = cma->base_pfn, pfn = base_pfn;  in cma_activate_area()
99 	unsigned i = cma->count >> pageblock_order;  in cma_activate_area()
102 	cma->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma), GFP_KERNEL);  in cma_activate_area()
103 	if (!cma->bitmap)  in cma_activate_area()
113 		for (j = pageblock_nr_pages; j; --j, pfn++) {  in cma_activate_area()
125 	} while (--i);  in cma_activate_area()
127 	mutex_init(&cma->lock);  in cma_activate_area()
130 	INIT_HLIST_HEAD(&cma->mem_head);  in cma_activate_area()
131 	spin_lock_init(&cma->mem_head_lock);  in cma_activate_area()
137 	bitmap_free(cma->bitmap);  in cma_activate_area()
139 	cma->count = 0;  in cma_activate_area()
140 	pr_err("CMA area %s could not be activated\n", cma->name);  in cma_activate_area()
156  * cma_init_reserved_mem() - create custom contiguous area from reserved memory
157  * @base: Base address of the reserved area
158  * @size: Size of the reserved area (in bytes),
165  * This function creates custom contiguous area from already reserved memory.
177 		pr_err("Not enough slots for CMA reserved regions!\n");  in cma_init_reserved_mem()
178 		return -ENOSPC;  in cma_init_reserved_mem()
182 		return -EINVAL;  in cma_init_reserved_mem()
186 			max_t(unsigned long, MAX_ORDER - 1, pageblock_order);  in cma_init_reserved_mem()
190 		return -EINVAL;  in cma_init_reserved_mem()
193 		return -EINVAL;  in cma_init_reserved_mem()
196 	 * Each reserved area must be initialised later, when more kernel  in cma_init_reserved_mem()
202 		snprintf(cma->name, CMA_MAX_NAME, name);  in cma_init_reserved_mem()
204 		snprintf(cma->name, CMA_MAX_NAME,  "cma%d\n", cma_area_count);  in cma_init_reserved_mem()
206 	cma->base_pfn = PFN_DOWN(base);  in cma_init_reserved_mem()
207 	cma->count = size >> PAGE_SHIFT;  in cma_init_reserved_mem()
208 	cma->order_per_bit = order_per_bit;  in cma_init_reserved_mem()
217  * cma_declare_contiguous_nid() - reserve custom contiguous area
218  * @base: Base address of the reserved area optional, use 0 for any
219  * @size: Size of the reserved area (in bytes),
220  * @limit: End address of the reserved memory (optional, 0 for any).
223  * @fixed: hint about where to place the reserved area
228  * This function reserves memory from early allocator. It should be
230  * has been activated and all other subsystems have already allocated/reserved
231  * memory. This function allows to create custom reserved areas.
252 	highmem_start = __pa(high_memory - 1) + 1;  in cma_declare_contiguous_nid()
257 		pr_err("Not enough slots for CMA reserved regions!\n");  in cma_declare_contiguous_nid()
258 		return -ENOSPC;  in cma_declare_contiguous_nid()
262 		return -EINVAL;  in cma_declare_contiguous_nid()
265 		return -EINVAL;  in cma_declare_contiguous_nid()
271 	 * you couldn't get a contiguous memory, which is not what we want.  in cma_declare_contiguous_nid()
274 			  max_t(unsigned long, MAX_ORDER - 1, pageblock_order));  in cma_declare_contiguous_nid()
275 	if (fixed && base & (alignment - 1)) {  in cma_declare_contiguous_nid()
276 		ret = -EINVAL;  in cma_declare_contiguous_nid()
283 	limit &= ~(alignment - 1);  in cma_declare_contiguous_nid()
290 		return -EINVAL;  in cma_declare_contiguous_nid()
294 	 * low/high memory boundary.  in cma_declare_contiguous_nid()
297 		ret = -EINVAL;  in cma_declare_contiguous_nid()
298 		pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",  in cma_declare_contiguous_nid()
312 		ret = -EINVAL;  in cma_declare_contiguous_nid()
318 	/* Reserve memory */  in cma_declare_contiguous_nid()
322 			ret = -EBUSY;  in cma_declare_contiguous_nid()
329 		 * All pages in the reserved area must come from the same zone.  in cma_declare_contiguous_nid()
330 		 * If the requested region crosses the low/high memory boundary,  in cma_declare_contiguous_nid()
331 		 * try allocating from high memory first and fall back to low  in cma_declare_contiguous_nid()
332 		 * memory in case of failure.  in cma_declare_contiguous_nid()
344 				ret = -ENOMEM;  in cma_declare_contiguous_nid()
361 	pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size / SZ_1M,  in cma_declare_contiguous_nid()
380 	mutex_lock(&cma->lock);  in cma_debug_show_areas()
383 		next_zero_bit = find_next_zero_bit(cma->bitmap, nbits, start);  in cma_debug_show_areas()
386 		next_set_bit = find_next_bit(cma->bitmap, nbits, next_zero_bit);  in cma_debug_show_areas()
387 		nr_zero = next_set_bit - next_zero_bit;  in cma_debug_show_areas()
388 		nr_part = nr_zero << cma->order_per_bit;  in cma_debug_show_areas()
394 	pr_cont("=> %lu free of %lu total pages\n", nr_total, cma->count);  in cma_debug_show_areas()
395 	mutex_unlock(&cma->lock);  in cma_debug_show_areas()
402  * cma_alloc() - allocate pages from contiguous area
403  * @cma:   Contiguous memory region for which the allocation is performed.
408  * This function allocates part of contiguous memory on specific
409  * contiguous memory area.
415 	unsigned long pfn = -1;  in cma_alloc()
420 	int ret = -ENOMEM;  in cma_alloc()
422 	if (!cma || !cma->count || !cma->bitmap)  in cma_alloc()
440 		mutex_lock(&cma->lock);  in cma_alloc()
441 		bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap,  in cma_alloc()
445 			mutex_unlock(&cma->lock);  in cma_alloc()
448 		bitmap_set(cma->bitmap, bitmap_no, bitmap_count);  in cma_alloc()
454 		mutex_unlock(&cma->lock);  in cma_alloc()
456 		pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);  in cma_alloc()
467 		if (ret != -EBUSY)  in cma_alloc()
470 		pr_debug("%s(): memory range at %p is busy, retrying\n",  in cma_alloc()
472 		/* try again with a bit different memory target */  in cma_alloc()
489 		pr_err("%s: alloc failed, req-size: %zu pages, ret: %d\n",  in cma_alloc()
499  * cma_release() - release allocated pages
500  * @cma:   Contiguous memory region for which the allocation is performed.
504  * This function releases memory allocated by cma_alloc().
519 	if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count)  in cma_release()
522 	VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);  in cma_release()