18 #define IS_ALIGNED_ORDER(x,order) IS_ALIGNED((x),BIT_ULL(order))  argument
102 	u8 metadata, order;  in split()  local
107 	order = metadata & ORDER_MASK;  in split()
108 	assert(IS_USABLE(metadata) && order && (order < NLISTS));  in split()
109 	assert(IS_ALIGNED_ORDER(pfn, order));  in split()
110 	assert(usable_area_contains_pfn(a, pfn + BIT(order) - 1));  in split()
116 	for (i = 0; i < BIT(order); i++) {  in split()
120 	if ((order == a->max_order) && (is_list_empty(a->freelists + order)))  in split()
122 	order--;  in split()
127 		list_add_tail(a->freelists + order, addr);  in split()
128 		list_add_tail(a->freelists + order, pfn_to_virt(pfn + BIT(order)));  in split()
131 		list_add(a->freelists + order, addr);  in split()
132 		list_add(a->freelists + order, pfn_to_virt(pfn + BIT(order)));  in split()
146 	u8 order;  in page_memalign_order()  local
150 	order = sz > al ? sz : al;  in page_memalign_order()
156 	order += fresh;  in page_memalign_order()
159 	for ( ; order <= a->max_order; order++) {  in page_memalign_order()
160 		p = fresh ? a->freelists[order].prev : a->freelists[order].next;  in page_memalign_order()
170 	if (order > a->max_order)  in page_memalign_order()
178 	for (; order > sz; order--)  in page_memalign_order()
184 		order--;  in page_memalign_order()
186 		memset(a->page_states + idx, STATUS_FRESH | order, BIT(order));  in page_memalign_order()
187 		list_add_tail(a->freelists + order, p);  in page_memalign_order()
188 		idx += BIT(order);  in page_memalign_order()
191 	memset(a->page_states + idx, STATUS_ALLOCATED | order, BIT(order));  in page_memalign_order()
220 static bool coalesce(struct mem_area *a, u8 order, pfn_t pfn, pfn_t pfn2)  in coalesce()  argument
224 	assert(IS_ALIGNED_ORDER(pfn, order) && IS_ALIGNED_ORDER(pfn2, order));  in coalesce()
225 	assert(pfn2 == pfn + BIT(order));  in coalesce()
229 	if (!usable_area_contains_pfn(a, pfn) || !usable_area_contains_pfn(a, pfn2 + BIT(order) - 1))  in coalesce()
234 	if ((a->page_states[first] != order) || (a->page_states[second] != order))  in coalesce()
241 	for (i = 0; i < (2ull << order); i++) {  in coalesce()
242 		assert(a->page_states[first + i] == order);  in coalesce()
243 		a->page_states[first + i] = order + 1;  in coalesce()
246 	list_add(a->freelists + order + 1, pfn_to_virt(pfn));  in coalesce()
247 	if (order + 1 > a->max_order)  in coalesce()
248 		a->max_order = order + 1;  in coalesce()
271 	u8 order;  in _free_pages()  local
282 	order = a->page_states[p] & ORDER_MASK;  in _free_pages()
287 	assert(order < NLISTS);  in _free_pages()
289 	assert(IS_ALIGNED_ORDER(pfn, order));  in _free_pages()
291 	assert(usable_area_contains_pfn(a, pfn + BIT(order) - 1));  in _free_pages()
293 	for (i = 0; i < BIT(order); i++) {  in _free_pages()
295 		assert(a->page_states[p + i] == (STATUS_ALLOCATED | order));  in _free_pages()
297 		a->page_states[p + i] = STATUS_FREE | order;  in _free_pages()
300 	list_add(a->freelists + order, mem);  in _free_pages()
307 		order = a->page_states[p] & ORDER_MASK;  in _free_pages()
313 		if (!IS_ALIGNED_ORDER(pfn, order + 1))  in _free_pages()
314 			pfn = pfn - BIT(order);  in _free_pages()
315 		pfn2 = pfn + BIT(order);  in _free_pages()
317 	} while (coalesce(a, order, pfn, pfn2));  in _free_pages()
413 void *alloc_pages_flags(unsigned int order, unsigned int flags)  in alloc_pages_flags()  argument
415 	return page_memalign_order_flags(order, order, flags);  in alloc_pages_flags()
469 	u8 order = 0;  in _page_alloc_init_area()  local
507 	for (i = a->base; i < a->top; i += 1ull << order) {  in _page_alloc_init_area()
509 		while (i + BIT(order) > a->top) {  in _page_alloc_init_area()
510 			assert(order);  in _page_alloc_init_area()
511 			order--;  in _page_alloc_init_area()
518 		while (IS_ALIGNED_ORDER(i, order + 1) && (i + BIT(order + 1) <= a->top))  in _page_alloc_init_area()
519 			order++;  in _page_alloc_init_area()
520 		assert(order < NLISTS);  in _page_alloc_init_area()
522 		memset(a->page_states + (i - a->base), STATUS_FRESH | order, BIT(order));  in _page_alloc_init_area()
523 		list_add(a->freelists + order, pfn_to_virt(i));  in _page_alloc_init_area()
524 		if (order > a->max_order)  in _page_alloc_init_area()
525 			a->max_order = order;  in _page_alloc_init_area()