linux-6.15/mm/percpu-vm.c

1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/percpu-vm.c - vmalloc area based chunk allocation
16 	/* must not be used on pre-mapped chunk */  in pcpu_chunk_page()
17 	WARN_ON(chunk->immutable);  in pcpu_chunk_page()
23  * pcpu_get_pages - get temp pages array
25  * Returns pointer to array of pointers to struct page which can be indexed
26  * with pcpu_page_idx().  Note that there is only one array and accesses
30  * Pointer to temp pages array on success.
45  * pcpu_free_pages - free pages which were allocated for @chunk
47  * @pages: array of pages to be freed, indexed by pcpu_page_idx()
71  * pcpu_alloc_pages - allocates pages for @chunk
73  * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
103 	while (--i >= page_start)  in pcpu_alloc_pages()
112 	return -ENOMEM;  in pcpu_alloc_pages()
116  * pcpu_pre_unmap_flush - flush cache prior to unmapping
141  * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
143  * @pages: pages array which can be used to pass information to free
168 				   page_end - page_start);  in pcpu_unmap_pages()
173  * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
201  * pcpu_map_pages - map pages into a pcpu_chunk
203  * @pages: pages array containing pages to be mapped
212  * reverse lookup (addr -> chunk).
223 				       page_end - page_start);  in pcpu_map_pages()
235 				   page_end - page_start);  in pcpu_map_pages()
244  * pcpu_post_map_flush - flush cache after mapping
264  * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
283 		return -ENOMEM;  in pcpu_populate_chunk()
286 		return -ENOMEM;  in pcpu_populate_chunk()
290 		return -ENOMEM;  in pcpu_populate_chunk()
298  * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
319 	 * successful population attempt so the temp pages array must  in pcpu_depopulate_chunk()
349 	chunk->data = vms;  in pcpu_create_chunk()
350 	chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];  in pcpu_create_chunk()
353 	trace_percpu_create_chunk(chunk->base_addr);  in pcpu_create_chunk()
364 	trace_percpu_destroy_chunk(chunk->base_addr);  in pcpu_destroy_chunk()
366 	if (chunk->data)  in pcpu_destroy_chunk()
367 		pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);  in pcpu_destroy_chunk()
383  * pcpu_should_reclaim_chunk - determine if a chunk should go into reclaim
403 	 * there is no system-wide shortage of empty pages aside from this  in pcpu_should_reclaim_chunk()
406 	return ((chunk->isolated && chunk->nr_empty_pop_pages) ||  in pcpu_should_reclaim_chunk()
408 		 (PCPU_EMPTY_POP_PAGES_HIGH + chunk->nr_empty_pop_pages) &&  in pcpu_should_reclaim_chunk()
409 		 chunk->nr_empty_pop_pages >= chunk->nr_pages / 4));  in pcpu_should_reclaim_chunk()