37 /* zero numa counters within a zone */
38 static void zero_zone_numa_counters(struct zone *zone)  in zero_zone_numa_counters()  argument
43 		atomic_long_set(&zone->vm_numa_event[item], 0);  in zero_zone_numa_counters()
45 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]  in zero_zone_numa_counters()
54 	struct zone *zone;  in zero_zones_numa_counters()  local
56 	for_each_populated_zone(zone)  in zero_zones_numa_counters()
57 		zero_zone_numa_counters(zone);  in zero_zones_numa_counters()
158  * Manage combined zone based / global counters
169 static void fold_vm_zone_numa_events(struct zone *zone)  in fold_vm_zone_numa_events()  argument
178 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in fold_vm_zone_numa_events()
184 		zone_numa_event_add(zone_numa_events[item], zone, item);  in fold_vm_zone_numa_events()
189 	struct zone *zone;  in fold_vm_numa_events()  local
191 	for_each_populated_zone(zone)  in fold_vm_numa_events()
192 		fold_vm_zone_numa_events(zone);  in fold_vm_numa_events()
198 int calculate_pressure_threshold(struct zone *zone)  in calculate_pressure_threshold()  argument
211 	watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);  in calculate_pressure_threshold()
222 int calculate_normal_threshold(struct zone *zone)  in calculate_normal_threshold()  argument
229 	 * of memory per zone. More memory means that we can defer updates for  in calculate_normal_threshold()
257 	mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);  in calculate_normal_threshold()
270  * Refresh the thresholds for each zone.
275 	struct zone *zone;  in refresh_zone_stat_thresholds()  local
286 	for_each_populated_zone(zone) {  in refresh_zone_stat_thresholds()
287 		struct pglist_data *pgdat = zone->zone_pgdat;  in refresh_zone_stat_thresholds()
290 		threshold = calculate_normal_threshold(zone);  in refresh_zone_stat_thresholds()
295 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in refresh_zone_stat_thresholds()
298 			/* Base nodestat threshold on the largest populated zone. */  in refresh_zone_stat_thresholds()
309 		tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);  in refresh_zone_stat_thresholds()
312 			zone->percpu_drift_mark = high_wmark_pages(zone) +  in refresh_zone_stat_thresholds()
318 				int (*calculate_pressure)(struct zone *))  in set_pgdat_percpu_threshold()  argument
320 	struct zone *zone;  in set_pgdat_percpu_threshold()  local
326 		zone = &pgdat->node_zones[i];  in set_pgdat_percpu_threshold()
327 		if (!zone->percpu_drift_mark)  in set_pgdat_percpu_threshold()
330 		threshold = (*calculate_pressure)(zone);  in set_pgdat_percpu_threshold()
332 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in set_pgdat_percpu_threshold()
342 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in __mod_zone_page_state()  argument
345 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __mod_zone_page_state()
364 		zone_page_state_add(x, zone, item);  in __mod_zone_page_state()
413  * argument instead of struct zone *. This allows the inclusion of the code
426  * However, the code must first determine the differential location in a zone
432 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)  in __inc_zone_state()  argument
434 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __inc_zone_state()
446 		zone_page_state_add(v + overstep, zone, item);  in __inc_zone_state()
488 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)  in __dec_zone_state()  argument
490 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __dec_zone_state()
502 		zone_page_state_add(v - overstep, zone, item);  in __dec_zone_state()
549  * mod_state() modifies the zone counter state through atomic per cpu
557 static inline void mod_zone_state(struct zone *zone,  in mod_zone_state()  argument
560 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in mod_zone_state()
567 		z = 0;  /* overflow to zone counters */  in mod_zone_state()
577 		 * for all cpus in a zone.  in mod_zone_state()
586 			/* Overflow must be added to zone counters */  in mod_zone_state()
593 		zone_page_state_add(z, zone, item);  in mod_zone_state()
596 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
599 	mod_zone_state(zone, item, delta, 0);  in mod_zone_page_state()
692 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
698 	__mod_zone_page_state(zone, item, delta);  in mod_zone_page_state()
706 	struct zone *zone;  in inc_zone_page_state()  local
708 	zone = page_zone(page);  in inc_zone_page_state()
710 	__inc_zone_state(zone, item);  in inc_zone_page_state()
793  * Update the zone counters for the current cpu.
802  * statistics in the remote zone struct as well as the global cachelines
811 	struct zone *zone;  in refresh_cpu_vm_stats()  local
817 	for_each_populated_zone(zone) {  in refresh_cpu_vm_stats()
818 		struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;  in refresh_cpu_vm_stats()
819 		struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;  in refresh_cpu_vm_stats()
827 				atomic_long_add(v, &zone->vm_stat[i]);  in refresh_cpu_vm_stats()
839 			changes += decay_pcp_high(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
855 			if (zone_to_nid(zone) == numa_node_id()) {  in refresh_cpu_vm_stats()
866 				drain_zone_pages(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
899 	struct zone *zone;  in cpu_vm_stats_fold()  local
904 	for_each_populated_zone(zone) {  in cpu_vm_stats_fold()
907 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in cpu_vm_stats_fold()
915 				atomic_long_add(v, &zone->vm_stat[i]);  in cpu_vm_stats_fold()
926 				zone_numa_event_add(v, zone, i);  in cpu_vm_stats_fold()
952  * this is only called if !populated_zone(zone), which implies no other users of
955 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)  in drain_zonestat()  argument
964 			zone_page_state_add(v, zone, i);  in drain_zonestat()
973 			zone_numa_event_add(v, zone, i);  in drain_zonestat()
989 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_node_page_state()
1003 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_numa_event_state()
1045  * Calculate the number of free pages in a zone, how many contiguous
1052 static void fill_contig_page_info(struct zone *zone,  in fill_contig_page_info()  argument
1071 		blocks = data_race(zone->free_area[order].nr_free);  in fill_contig_page_info()
1115  * Calculates external fragmentation within a zone wrt the given order.
1119 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)  in extfrag_for_order()  argument
1123 	fill_contig_page_info(zone, order, &info);  in extfrag_for_order()
1133 int fragmentation_index(struct zone *zone, unsigned int order)  in fragmentation_index()  argument
1137 	fill_contig_page_info(zone, order, &info);  in fragmentation_index()
1453 		void (*print)(struct seq_file *m, pg_data_t *, struct zone *))  in walk_zones_in_node()  argument
1455 	struct zone *zone;  in walk_zones_in_node()  local
1456 	struct zone *node_zones = pgdat->node_zones;  in walk_zones_in_node()
1459 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {  in walk_zones_in_node()
1460 		if (assert_populated && !populated_zone(zone))  in walk_zones_in_node()
1464 			spin_lock_irqsave(&zone->lock, flags);  in walk_zones_in_node()
1465 		print(m, pgdat, zone);  in walk_zones_in_node()
1467 			spin_unlock_irqrestore(&zone->lock, flags);  in walk_zones_in_node()
1474 						struct zone *zone)  in frag_show_print()  argument
1478 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in frag_show_print()
1484 		seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));  in frag_show_print()
1489  * This walks the free areas for each zone.
1499 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showfree_print()  argument
1504 		seq_printf(m, "Node %4d, zone %8s, type %12s ",  in pagetypeinfo_showfree_print()
1506 					zone->name,  in pagetypeinfo_showfree_print()
1514 			area = &(zone->free_area[order]);  in pagetypeinfo_showfree_print()
1532 			spin_unlock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1534 			spin_lock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1556 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showblockcount_print()  argument
1560 	unsigned long start_pfn = zone->zone_start_pfn;  in pagetypeinfo_showblockcount_print()
1561 	unsigned long end_pfn = zone_end_pfn(zone);  in pagetypeinfo_showblockcount_print()
1571 		if (page_zone(page) != zone)  in pagetypeinfo_showblockcount_print()
1581 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in pagetypeinfo_showblockcount_print()
1663 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)  in is_zone_first_populated()  argument
1668 		struct zone *compare = &pgdat->node_zones[zid];  in is_zone_first_populated()
1671 			return zone == compare;  in is_zone_first_populated()
1678 							struct zone *zone)  in zoneinfo_show_print()  argument
1681 	seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);  in zoneinfo_show_print()
1682 	if (is_zone_first_populated(pgdat, zone)) {  in zoneinfo_show_print()
1703 		   zone_page_state(zone, NR_FREE_PAGES),  in zoneinfo_show_print()
1704 		   zone->watermark_boost,  in zoneinfo_show_print()
1705 		   min_wmark_pages(zone),  in zoneinfo_show_print()
1706 		   low_wmark_pages(zone),  in zoneinfo_show_print()
1707 		   high_wmark_pages(zone),  in zoneinfo_show_print()
1708 		   zone->spanned_pages,  in zoneinfo_show_print()
1709 		   zone->present_pages,  in zoneinfo_show_print()
1710 		   zone_managed_pages(zone),  in zoneinfo_show_print()
1711 		   zone_cma_pages(zone));  in zoneinfo_show_print()
1715 		   zone->lowmem_reserve[0]);  in zoneinfo_show_print()
1716 	for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)  in zoneinfo_show_print()
1717 		seq_printf(m, ", %ld", zone->lowmem_reserve[i]);  in zoneinfo_show_print()
1721 	if (!populated_zone(zone)) {  in zoneinfo_show_print()
1728 			   zone_page_state(zone, i));  in zoneinfo_show_print()
1733 			   zone_numa_event_state(zone, i));  in zoneinfo_show_print()
1741 		pcp = per_cpu_ptr(zone->per_cpu_pageset, i);  in zoneinfo_show_print()
1752 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);  in zoneinfo_show_print()
1761 		   zone->zone_start_pfn);  in zoneinfo_show_print()
1967 	struct zone *zone;  in need_update()  local
1969 	for_each_populated_zone(zone) {  in need_update()
1970 		struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in need_update()
1979 		if (last_pgdat == zone->zone_pgdat)  in need_update()
1981 		last_pgdat = zone->zone_pgdat;  in need_update()
1982 		n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);  in need_update()
2175 					pg_data_t *pgdat, struct zone *zone)  in unusable_show_print()  argument
2181 	seq_printf(m, "Node %d, zone %8s ",  in unusable_show_print()
2183 				zone->name);  in unusable_show_print()
2185 		fill_contig_page_info(zone, order, &info);  in unusable_show_print()
2225 					pg_data_t *pgdat, struct zone *zone)  in extfrag_show_print()  argument
2230 	/* Alloc on stack as interrupts are disabled for zone walk */  in extfrag_show_print()
2233 	seq_printf(m, "Node %d, zone %8s ",  in extfrag_show_print()
2235 				zone->name);  in extfrag_show_print()
2237 		fill_contig_page_info(zone, order, &info);  in extfrag_show_print()