1 // SPDX-License-Identifier: GPL-2.0
23  * 1) mem_section	- memory sections, mem_map's for valid memory
64 	struct mem_section *section = NULL;  in sparse_index_alloc()  local
69 		section = kzalloc_node(array_size, GFP_KERNEL, nid);  in sparse_index_alloc()
71 		section = memblock_alloc_node(array_size, SMP_CACHE_BYTES,  in sparse_index_alloc()
73 		if (!section)  in sparse_index_alloc()
78 	return section;  in sparse_index_alloc()
84 	struct mem_section *section;  in sparse_index_init()  local
87 	 * An existing section is possible in the sub-section hotplug  in sparse_index_init()
88 	 * case. First hot-add instantiates, follow-on hot-add reuses  in sparse_index_init()
89 	 * the existing section.  in sparse_index_init()
96 	section = sparse_index_alloc(nid);  in sparse_index_init()
97 	if (!section)  in sparse_index_init()
98 		return -ENOMEM;  in sparse_index_init()
100 	mem_section[root] = section;  in sparse_index_init()
128 	return (root_nr * SECTIONS_PER_ROOT) + (ms - root);  in __section_nr()
133 	return (unsigned long)(ms - mem_section[0]);  in __section_nr()
139  * mem_map, we use section_mem_map to store the section's NUMA
148 static inline int sparse_early_nid(struct mem_section *section)  in sparse_early_nid()  argument
150 	return (section->section_mem_map >> SECTION_NID_SHIFT);  in sparse_early_nid()
157 	unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT);  in mminit_validate_memmodel_limits()
160 	 * Sanity checks - do not allow an architecture to pass  in mminit_validate_memmodel_limits()
165 			"Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",  in mminit_validate_memmodel_limits()
172 			"End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",  in mminit_validate_memmodel_limits()
196 	ms->section_mem_map |= SECTION_MARKED_PRESENT;  in section_mark_present()
200 	for (section_nr = next_present_section_nr(start-1);	\
201 	     ((section_nr != -1) &&				\
207 	return next_present_section_nr(-1);  in first_present_section_nr()
215 	int end = subsection_map_index(pfn + nr_pages - 1);  in subsection_mask_set()
217 	bitmap_set(map, idx, end - idx + 1);  in subsection_mask_set()
222 	int end_sec = pfn_to_section_nr(pfn + nr_pages - 1);  in subsection_map_init()
233 				- (pfn & ~PAGE_SECTION_MASK));  in subsection_map_init()
235 		subsection_mask_set(ms->usage->subsection_map, pfn, pfns);  in subsection_map_init()
239 				subsection_map_index(pfn + pfns - 1));  in subsection_map_init()
242 		nr_pages -= pfns;  in subsection_map_init()
272 		unsigned long section = pfn_to_section_nr(pfn);  in memory_present()  local
275 		sparse_index_init(section, nid);  in memory_present()
276 		set_section_nid(section, nid);  in memory_present()
278 		ms = __nr_to_section(section);  in memory_present()
279 		if (!ms->section_mem_map) {  in memory_present()
280 			ms->section_mem_map = sparse_encode_early_nid(nid) |  in memory_present()
303  * the identity pfn - section_mem_map will return the actual
309 		(unsigned long)(mem_map - (section_nr_to_pfn(pnum)));  in sparse_encode_mem_map()
331 	ms->section_mem_map &= ~SECTION_MAP_MASK;  in sparse_init_one_section()
332 	ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum)  in sparse_init_one_section()
334 	ms->usage = usage;  in sparse_init_one_section()
357 	 * page being freed and making a section unremovable while  in sparse_early_usemaps_alloc_pgdat_section()
359 	 * a pgdat can prevent a section being removed. If section A  in sparse_early_usemaps_alloc_pgdat_section()
360 	 * contains a pgdat and section B contains the usemap, both  in sparse_early_usemaps_alloc_pgdat_section()
361 	 * sections become inter-dependent. This allocates usemaps  in sparse_early_usemaps_alloc_pgdat_section()
362 	 * from the same section as the pgdat where possible to avoid  in sparse_early_usemaps_alloc_pgdat_section()
406 		pr_info("node %d must be removed before remove section %ld\n",  in check_usemap_section_nr()
412 	 * Some platforms allow un-removable section because they will just  in check_usemap_section_nr()
414 	 * Just notify un-removable section's number here.  in check_usemap_section_nr()
416 …pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\…  in check_usemap_section_nr()
424 	return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id);  in sparse_early_usemaps_alloc_pgdat_section()
479 	 * Pre-allocated buffer is mainly used by __populate_section_memmap  in sparse_buffer_init()
480 	 * and we want it to be properly aligned to the section size - this is  in sparse_buffer_init()
490 	unsigned long size = sparsemap_buf_end - sparsemap_buf;  in sparse_buffer_fini()
507 			if ((unsigned long)(ptr - sparsemap_buf) > 0)  in sparse_buffer_alloc()
508 				sparse_buffer_free((unsigned long)(ptr - sparsemap_buf));  in sparse_buffer_alloc()
567 		ms->section_mem_map = 0;  in sparse_init_nid()
572  * Allocate the accumulated non-linear sections, allocate a mem_map
573  * for each and record the physical to section mapping.
622 		ms->section_mem_map |= SECTION_IS_ONLINE;  in online_mem_sections()
644 		ms->section_mem_map &= ~SECTION_IS_ONLINE;  in offline_mem_sections()
677 	unsigned long *subsection_map = ms->usage  in clear_subsection_map()
678 		? &ms->usage->subsection_map[0] : NULL;  in clear_subsection_map()
685 				"section already deactivated (%#lx + %ld)\n",  in clear_subsection_map()
687 		return -EINVAL;  in clear_subsection_map()
695 	return bitmap_empty(&ms->usage->subsection_map[0],  in is_subsection_map_empty()
708 	subsection_map = &ms->usage->subsection_map[0];  in fill_subsection_map()
711 		rc = -EINVAL;  in fill_subsection_map()
713 		rc = -EEXIST;  in fill_subsection_map()
744 		magic = (unsigned long) page->freelist;  in free_map_bootmem()
752 		 * When this function is called, the removing section is  in free_map_bootmem()
754 		 * from page allocator. If removing section's memmap is placed  in free_map_bootmem()
755 		 * on the same section, it must not be freed.  in free_map_bootmem()
784  * 1. deactivation of a partial hot-added section (only possible in
786  *      a) section was present at memory init.
787  *      b) section was hot-added post memory init.
788  * 2. deactivation of a complete hot-added section.
789  * 3. deactivation of a complete section from memory init.
812 		 * When removing an early section, the usage map is kept (as the  in section_deactivate()
814 		 * will be re-used when re-adding the section - which is then no  in section_deactivate()
815 		 * longer an early section. If the usage map is PageReserved, it  in section_deactivate()
818 		if (!PageReserved(virt_to_page(ms->usage))) {  in section_deactivate()
819 			kfree(ms->usage);  in section_deactivate()
820 			ms->usage = NULL;  in section_deactivate()
822 		memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);  in section_deactivate()
824 		 * Mark the section invalid so that valid_section()  in section_deactivate()
826 		 * ms->usage array.  in section_deactivate()
828 		ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;  in section_deactivate()
841 		ms->section_mem_map = (unsigned long)NULL;  in section_deactivate()
852 	if (!ms->usage) {  in section_activate()
855 			return ERR_PTR(-ENOMEM);  in section_activate()
856 		ms->usage = usage;  in section_activate()
862 			ms->usage = NULL;  in section_activate()
870 	 * referenced.  If we hot-add memory into such a section then we  in section_activate()
880 		return ERR_PTR(-ENOMEM);  in section_activate()
887  * sparse_add_section - add a memory section, or populate an existing one
888  * @nid: The node to add section on
890  * @nr_pages: number of pfns to add in the section
895  * Note that only VMEMMAP supports sub-section aligned hotplug,
900  * * 0		- On success.
901  * * -EEXIST	- Section has been present.
902  * * -ENOMEM	- Out of memory.
930 	/* Align memmap to section boundary in the subsection case */  in sparse_add_section()
933 	sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0);  in sparse_add_section()
944 	 * A further optimization is to have per section refcounted  in clear_hwpoisoned_pages()
970 			nr_pages - map_offset);  in sparse_remove_section()