68 		struct zone *zone;  in mminit_verify_zonelist()  local
75 			/* Identify the zone and nodelist */  in mminit_verify_zonelist()
79 			zone = &pgdat->node_zones[zoneid];  in mminit_verify_zonelist()
80 			if (!populated_zone(zone))  in mminit_verify_zonelist()
86 				zone->name);  in mminit_verify_zonelist()
89 			for_each_zone_zonelist(zone, z, zonelist, zoneid)  in mminit_verify_zonelist()
90 				pr_cont("%d:%s ", zone_to_nid(zone), zone->name);  in mminit_verify_zonelist()
104 		"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",  in mminit_verify_pageflags_layout()
114 		"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",  in mminit_verify_pageflags_layout()
121 		"Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",  in mminit_verify_pageflags_layout()
128 		"Node/Zone ID: %lu -> %lu\n",  in mminit_verify_pageflags_layout()
310  * Sum pages in active regions for movable zone.
330  * This finds a zone that can be used for ZONE_MOVABLE pages. The
332  * increasing memory addresses so that the "highest" populated zone is used
351  * Find the PFN the Movable zone begins in each node. Kernel memory
581 				unsigned long zone, int nid)  in __init_single_page()  argument
584 	set_page_links(page, zone, nid, pfn);  in __init_single_page()
593 	if (!is_highmem_idx(zone))  in __init_single_page()
669  * Initialize a reserved page unconditionally, finding its zone first.
679 		struct zone *zone = &pgdat->node_zones[zid];  in __init_page_from_nid()  local
681 		if (zone_spans_pfn(zone, pfn))  in __init_page_from_nid()
725 	 * prev_end_pfn static that contains the end of previous zone  in defer_init()
799 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
801 overlap_memmap_init(unsigned long zone, unsigned long *pfn)  in overlap_memmap_init()  argument
805 	if (mirrored_kernelcore && zone == ZONE_MOVABLE) {  in overlap_memmap_init()
837  * - zone and node links point to zone and node that span the page if the
838  *   hole is in the middle of a zone
839  * - zone and node links point to adjacent zone/node if the hole falls on
840  *   the zone boundary; the pages in such holes will be prepended to the
841  *   zone/node above the hole except for the trailing pages in the last
842  *   section that will be appended to the zone/node below.
846 					  int zone, int node)  in init_unavailable_range()  argument
856 		__init_single_page(pfn_to_page(pfn), pfn, zone, node);  in init_unavailable_range()
862 		pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n",  in init_unavailable_range()
863 			node, zone_names[zone], pgcnt);  in init_unavailable_range()
873  * zone stats (e.g., nr_isolate_pageblock) are touched.
875 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,  in memmap_init_range()  argument
894 	if (zone == ZONE_DEVICE) {  in memmap_init_range()
910 			if (overlap_memmap_init(zone, &pfn))  in memmap_init_range()
919 		__init_single_page(page, pfn, zone, nid);  in memmap_init_range()
922 			if (zone == ZONE_DEVICE)  in memmap_init_range()
942 static void __init memmap_init_zone_range(struct zone *zone,  in memmap_init_zone_range()  argument
947 	unsigned long zone_start_pfn = zone->zone_start_pfn;  in memmap_init_zone_range()
948 	unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;  in memmap_init_zone_range()
949 	int nid = zone_to_nid(zone), zone_id = zone_idx(zone);  in memmap_init_zone_range()
976 			struct zone *zone = node->node_zones + j;  in memmap_init()  local
978 			if (!populated_zone(zone))  in memmap_init()
981 			memmap_init_zone_range(zone, start_pfn, end_pfn,  in memmap_init()
991 	 * Append the pages in this hole to the highest zone in the last  in memmap_init()
1013 	 * phase for it to be fully associated with a zone.  in __init_zone_device_page()
1109 void __ref memmap_init_zone_device(struct zone *zone,  in memmap_init_zone_device()  argument
1115 	struct pglist_data *pgdat = zone->zone_pgdat;  in memmap_init_zone_device()
1118 	unsigned long zone_idx = zone_idx(zone);  in memmap_init_zone_device()
1153  * The zone ranges provided by the architecture do not include ZONE_MOVABLE
1157  * is distributed. This helper function adjusts the zone ranges
1159  * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
1221 /* Return the number of page frames in holes in a zone on a node */
1229 	/* zone is empty, we don't have any absent pages */  in zone_absent_pages_in_node()
1264  * Return the number of pages a zone spans in a node, including holes
1277 	/* Get the start and end of the zone */  in zone_spanned_pages_in_node()
1283 	/* Check that this node has pages within the zone's required range */  in zone_spanned_pages_in_node()
1287 	/* Move the zone boundaries inside the node if necessary */  in zone_spanned_pages_in_node()
1297 	struct zone *z;  in reset_memoryless_node_totalpages()
1345 		struct zone *zone = pgdat->node_zones + i;  in calculate_node_totalpages()  local
1362 			zone->zone_start_pfn = zone_start_pfn;  in calculate_node_totalpages()
1364 			zone->zone_start_pfn = 0;  in calculate_node_totalpages()
1365 		zone->spanned_pages = spanned;  in calculate_node_totalpages()
1366 		zone->present_pages = real_size;  in calculate_node_totalpages()
1368 		zone->present_early_pages = real_size;  in calculate_node_totalpages()
1422 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,  in zone_init_internals()  argument
1425 	atomic_long_set(&zone->managed_pages, remaining_pages);  in zone_init_internals()
1426 	zone_set_nid(zone, nid);  in zone_init_internals()
1427 	zone->name = zone_names[idx];  in zone_init_internals()
1428 	zone->zone_pgdat = NODE_DATA(nid);  in zone_init_internals()
1429 	spin_lock_init(&zone->lock);  in zone_init_internals()
1430 	zone_seqlock_init(zone);  in zone_init_internals()
1431 	zone_pcp_init(zone);  in zone_init_internals()
1434 static void __meminit zone_init_free_lists(struct zone *zone)  in zone_init_free_lists()  argument
1438 		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);  in zone_init_free_lists()
1439 		zone->free_area[order].nr_free = 0;  in zone_init_free_lists()
1443 	INIT_LIST_HEAD(&zone->unaccepted_pages);  in zone_init_free_lists()
1447 void __meminit init_currently_empty_zone(struct zone *zone,  in init_currently_empty_zone()  argument
1451 	struct pglist_data *pgdat = zone->zone_pgdat;  in init_currently_empty_zone()
1452 	int zone_idx = zone_idx(zone) + 1;  in init_currently_empty_zone()
1457 	zone->zone_start_pfn = zone_start_pfn;  in init_currently_empty_zone()
1460 			"Initialising map node %d zone %lu pfns %lu -> %lu\n",  in init_currently_empty_zone()
1462 			(unsigned long)zone_idx(zone),  in init_currently_empty_zone()
1465 	zone_init_free_lists(zone);  in init_currently_empty_zone()
1466 	zone->initialized = 1;  in init_currently_empty_zone()
1471  * Calculate the size of the zone->pageblock_flags rounded to an unsigned long
1490 static void __ref setup_usemap(struct zone *zone)  in setup_usemap()  argument
1492 	unsigned long usemapsize = usemap_size(zone->zone_start_pfn,  in setup_usemap()
1493 					       zone->spanned_pages);  in setup_usemap()
1494 	zone->pageblock_flags = NULL;  in setup_usemap()
1496 		zone->pageblock_flags =  in setup_usemap()
1498 					    zone_to_nid(zone));  in setup_usemap()
1499 		if (!zone->pageblock_flags)  in setup_usemap()
1500 			panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",  in setup_usemap()
1501 			      usemapsize, zone->name, zone_to_nid(zone));  in setup_usemap()
1505 static inline void setup_usemap(struct zone *zone) {}  in setup_usemap()  argument
1544  * Set up the zone data structures
1586 		struct zone *zone = pgdat->node_zones + z;  in free_area_init_core_hotplug()  local
1588 		zone->present_pages = 0;  in free_area_init_core_hotplug()
1589 		zone_init_internals(zone, z, nid, 0);  in free_area_init_core_hotplug()
1603 		struct zone *zone = pgdat->node_zones + j;  in free_area_init_core()  local
1604 		unsigned long size = zone->spanned_pages;  in free_area_init_core()
1607 		 * Initialize zone->managed_pages as 0 , it will be reset  in free_area_init_core()
1610 		zone_init_internals(zone, j, nid, zone->present_pages);  in free_area_init_core()
1615 		setup_usemap(zone);  in free_area_init_core()
1616 		init_currently_empty_zone(zone, zone->zone_start_pfn, size);  in free_area_init_core()
1657 	 * The zone's endpoints aren't required to be MAX_PAGE_ORDER  in alloc_node_mem_map()
1755 		struct zone *zone = &pgdat->node_zones[zone_type];  in check_for_memory()  local
1756 		if (populated_zone(zone)) {  in check_for_memory()
1810  * free_area_init - Initialise all pg_data_t and zone data
1811  * @max_zone_pfn: an array of max PFNs for each zone
1815  * zone in each node and their holes is calculated. If the maximum PFN
1816  * between two adjacent zones match, it is assumed that the zone is empty.
1818  * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
1825 	int i, nid, zone;  in free_area_init()  local
1828 	/* Record where the zone boundaries are */  in free_area_init()
1839 			zone = MAX_NR_ZONES - i - 1;  in free_area_init()
1841 			zone = i;  in free_area_init()
1843 		if (zone == ZONE_MOVABLE)  in free_area_init()
1846 		end_pfn = max(max_zone_pfn[zone], start_pfn);  in free_area_init()
1847 		arch_zone_lowest_possible_pfn[zone] = start_pfn;  in free_area_init()
1848 		arch_zone_highest_possible_pfn[zone] = end_pfn;  in free_area_init()
1857 	/* Print out the zone ranges */  in free_area_init()
1858 	pr_info("Zone ranges:\n");  in free_area_init()
1875 	pr_info("Movable zone start for each node\n");  in free_area_init()
2030 static unsigned long __init deferred_init_pages(struct zone *zone,  in deferred_init_pages()  argument
2033 	int nid = zone_to_nid(zone);  in deferred_init_pages()
2035 	int zid = zone_idx(zone);  in deferred_init_pages()
2044  * This function is meant to pre-load the iterator for the zone init from
2052 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,  in deferred_init_mem_pfn_range_in_zone()  argument
2059 		__next_mem_pfn_range_in_zone(&j, zone, spfn, epfn);  in deferred_init_mem_pfn_range_in_zone()
2062 	 * Start out by walking through the ranges in this zone that have  in deferred_init_mem_pfn_range_in_zone()
2066 	for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) {  in deferred_init_mem_pfn_range_in_zone()
2089 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,  in deferred_init_maxorder()  argument
2098 	for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {  in deferred_init_maxorder()
2105 		nr_pages += deferred_init_pages(zone, *start_pfn, t);  in deferred_init_maxorder()
2116 	for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {  in deferred_init_maxorder()
2137 	struct zone *zone = arg;  in deferred_init_memmap_chunk()  local
2140 	deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);  in deferred_init_memmap_chunk()
2147 		deferred_init_maxorder(&i, zone, &spfn, &epfn);  in deferred_init_memmap_chunk()
2166 	struct zone *zone;  in deferred_init_memmap()  local
2188 	 * Once we unlock here, the zone cannot be grown anymore, thus if an  in deferred_init_memmap()
2189 	 * interrupt thread must allocate this early in boot, zone must be  in deferred_init_memmap()
2194 	/* Only the highest zone is deferred */  in deferred_init_memmap()
2195 	zone = pgdat->node_zones + pgdat->nr_zones - 1;  in deferred_init_memmap()
2199 	while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_init_pfn)) {  in deferred_init_memmap()
2203 			.fn_arg      = zone,  in deferred_init_memmap()
2215 	/* Sanity check that the next zone really is unpopulated */  in deferred_init_memmap()
2216 	WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone));  in deferred_init_memmap()
2226  * If this zone has deferred pages, try to grow it by initializing enough
2232  * Return true when zone was grown, otherwise return false. We return true even
2236 bool __init deferred_grow_zone(struct zone *zone, unsigned int order)  in deferred_grow_zone()  argument
2239 	pg_data_t *pgdat = zone->zone_pgdat;  in deferred_grow_zone()
2245 	/* Only the last zone may have deferred pages */  in deferred_grow_zone()
2246 	if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))  in deferred_grow_zone()
2252 	 * If someone grew this zone while we were waiting for spinlock, return  in deferred_grow_zone()
2260 	/* If the zone is empty somebody else may have cleared out the zone */  in deferred_grow_zone()
2261 	if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,  in deferred_grow_zone()
2278 		nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);  in deferred_grow_zone()
2329 void set_zone_contiguous(struct zone *zone)  in set_zone_contiguous()  argument
2331 	unsigned long block_start_pfn = zone->zone_start_pfn;  in set_zone_contiguous()
2335 	for (; block_start_pfn < zone_end_pfn(zone);  in set_zone_contiguous()
2339 		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));  in set_zone_contiguous()
2342 					     block_end_pfn, zone))  in set_zone_contiguous()
2348 	zone->contiguous = true;  in set_zone_contiguous()
2359 	struct zone *zone, *izone = NULL;  in pfn_range_intersects_zones()  local
2361 	for_each_zone(zone) {  in pfn_range_intersects_zones()
2362 		if (nid != NUMA_NO_NODE && zone_to_nid(zone) != nid)  in pfn_range_intersects_zones()
2365 		if (zone_intersects(zone, start_pfn, nr_pages)) {  in pfn_range_intersects_zones()
2368 			izone = zone;  in pfn_range_intersects_zones()
2379 	struct zone *zone;  in page_alloc_init_late()  local
2413 	for_each_populated_zone(zone)  in page_alloc_init_late()
2414 		set_zone_contiguous(zone);  in page_alloc_init_late()