Lines Matching full:zone
13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
80 * shuffle the whole zone).
89 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
293 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
306 static bool try_to_accept_memory(struct zone *zone, unsigned int order);
332 _deferred_grow_zone(struct zone *zone, unsigned int order) in _deferred_grow_zone() argument
334 return deferred_grow_zone(zone, order); in _deferred_grow_zone()
442 static int page_outside_zone_boundaries(struct zone *zone, struct page *page) in page_outside_zone_boundaries() argument
450 seq = zone_span_seqbegin(zone); in page_outside_zone_boundaries()
451 start_pfn = zone->zone_start_pfn; in page_outside_zone_boundaries()
452 sp = zone->spanned_pages; in page_outside_zone_boundaries()
453 ret = !zone_spans_pfn(zone, pfn); in page_outside_zone_boundaries()
454 } while (zone_span_seqretry(zone, seq)); in page_outside_zone_boundaries()
457 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", in page_outside_zone_boundaries()
458 pfn, zone_to_nid(zone), zone->name, in page_outside_zone_boundaries()
465 * Temporary debugging check for pages not lying within a given zone.
467 static int __maybe_unused bad_range(struct zone *zone, struct page *page) in bad_range() argument
469 if (page_outside_zone_boundaries(zone, page)) in bad_range()
471 if (zone != page_zone(page)) in bad_range()
477 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page) in bad_range() argument
613 static inline struct capture_control *task_capc(struct zone *zone) in task_capc() argument
620 capc->cc->zone == zone ? capc : NULL; in task_capc()
649 static inline struct capture_control *task_capc(struct zone *zone) in task_capc() argument
663 static inline void add_to_free_list(struct page *page, struct zone *zone, in add_to_free_list() argument
666 struct free_area *area = &zone->free_area[order]; in add_to_free_list()
673 static inline void add_to_free_list_tail(struct page *page, struct zone *zone, in add_to_free_list_tail() argument
676 struct free_area *area = &zone->free_area[order]; in add_to_free_list_tail()
687 static inline void move_to_free_list(struct page *page, struct zone *zone, in move_to_free_list() argument
690 struct free_area *area = &zone->free_area[order]; in move_to_free_list()
695 static inline void del_page_from_free_list(struct page *page, struct zone *zone, in del_page_from_free_list() argument
705 zone->free_area[order].nr_free--; in del_page_from_free_list()
766 struct zone *zone, unsigned int order, in __free_one_page() argument
769 struct capture_control *capc = task_capc(zone); in __free_one_page()
775 VM_BUG_ON(!zone_is_initialized(zone)); in __free_one_page()
780 __mod_zone_freepage_state(zone, 1 << order, migratetype); in __free_one_page()
783 VM_BUG_ON_PAGE(bad_range(zone, page), page); in __free_one_page()
787 __mod_zone_freepage_state(zone, -(1 << order), in __free_one_page()
816 clear_page_guard(zone, buddy, order, migratetype); in __free_one_page()
818 del_page_from_free_list(buddy, zone, order); in __free_one_page()
836 add_to_free_list_tail(page, zone, order, migratetype); in __free_one_page()
838 add_to_free_list(page, zone, order, migratetype); in __free_one_page()
861 struct zone *zone = page_zone(free_page); in split_free_page() local
872 spin_lock_irqsave(&zone->lock, flags); in split_free_page()
881 __mod_zone_freepage_state(zone, -(1UL << order), mt); in split_free_page()
883 del_page_from_free_list(free_page, zone, order); in split_free_page()
891 __free_one_page(pfn_to_page(pfn), pfn, zone, free_page_order, in split_free_page()
900 spin_unlock_irqrestore(&zone->lock, flags); in split_free_page()
1184 * Assumes all pages on list are in same zone.
1187 static void free_pcppages_bulk(struct zone *zone, int count, in free_pcppages_bulk() argument
1205 spin_lock_irqsave(&zone->lock, flags); in free_pcppages_bulk()
1206 isolated_pageblocks = has_isolate_pageblock(zone); in free_pcppages_bulk()
1238 __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE); in free_pcppages_bulk()
1243 spin_unlock_irqrestore(&zone->lock, flags); in free_pcppages_bulk()
1246 static void free_one_page(struct zone *zone, in free_one_page() argument
1253 spin_lock_irqsave(&zone->lock, flags); in free_one_page()
1254 if (unlikely(has_isolate_pageblock(zone) || in free_one_page()
1258 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags); in free_one_page()
1259 spin_unlock_irqrestore(&zone->lock, flags); in free_one_page()
1267 struct zone *zone = page_zone(page); in __free_pages_ok() local
1279 free_one_page(zone, page, pfn, order, migratetype, fpi_flags); in __free_pages_ok()
1322 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1329 * belong to a single zone. We assume that a border between node0 and node1
1345 unsigned long end_pfn, struct zone *zone) in __pageblock_pfn_to_page() argument
1360 if (page_zone(start_page) != zone) in __pageblock_pfn_to_page()
1386 static inline void expand(struct zone *zone, struct page *page, in expand() argument
1394 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); in expand()
1402 if (set_page_guard(zone, &page[size], high, migratetype)) in expand()
1405 add_to_free_list(&page[size], zone, high, migratetype); in expand()
1562 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, in __rmqueue_smallest() argument
1571 area = &(zone->free_area[current_order]); in __rmqueue_smallest()
1575 del_page_from_free_list(page, zone, current_order); in __rmqueue_smallest()
1576 expand(zone, page, order, current_order, migratetype); in __rmqueue_smallest()
1601 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone, in __rmqueue_cma_fallback() argument
1604 return __rmqueue_smallest(zone, order, MIGRATE_CMA); in __rmqueue_cma_fallback()
1607 static inline struct page *__rmqueue_cma_fallback(struct zone *zone, in __rmqueue_cma_fallback() argument
1616 static int move_freepages(struct zone *zone, in move_freepages() argument
1641 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); in move_freepages()
1642 VM_BUG_ON_PAGE(page_zone(page) != zone, page); in move_freepages()
1645 move_to_free_list(page, zone, order, migratetype); in move_freepages()
1653 int move_freepages_block(struct zone *zone, struct page *page, in move_freepages_block() argument
1665 /* Do not cross zone boundaries */ in move_freepages_block()
1666 if (!zone_spans_pfn(zone, start_pfn)) in move_freepages_block()
1668 if (!zone_spans_pfn(zone, end_pfn)) in move_freepages_block()
1671 return move_freepages(zone, start_pfn, end_pfn, migratetype, in move_freepages_block()
1719 static inline bool boost_watermark(struct zone *zone) in boost_watermark() argument
1731 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone)) in boost_watermark()
1734 max_boost = mult_frac(zone->_watermark[WMARK_HIGH], in boost_watermark()
1750 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages, in boost_watermark()
1764 static void steal_suitable_fallback(struct zone *zone, struct page *page, in steal_suitable_fallback() argument
1791 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD)) in steal_suitable_fallback()
1792 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); in steal_suitable_fallback()
1798 free_pages = move_freepages_block(zone, page, start_type, in steal_suitable_fallback()
1800 /* moving whole block can fail due to zone boundary conditions */ in steal_suitable_fallback()
1836 move_to_free_list(page, zone, current_order, start_type); in steal_suitable_fallback()
1877 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone) in reserve_highatomic_pageblock() argument
1884 * roughly 1% of a zone. But if 1% of a zone falls below a in reserve_highatomic_pageblock()
1888 if ((zone_managed_pages(zone) / 100) < pageblock_nr_pages) in reserve_highatomic_pageblock()
1890 max_managed = ALIGN((zone_managed_pages(zone) / 100), pageblock_nr_pages); in reserve_highatomic_pageblock()
1891 if (zone->nr_reserved_highatomic >= max_managed) in reserve_highatomic_pageblock()
1894 spin_lock_irqsave(&zone->lock, flags); in reserve_highatomic_pageblock()
1897 if (zone->nr_reserved_highatomic >= max_managed) in reserve_highatomic_pageblock()
1904 zone->nr_reserved_highatomic += pageblock_nr_pages; in reserve_highatomic_pageblock()
1906 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL); in reserve_highatomic_pageblock()
1910 spin_unlock_irqrestore(&zone->lock, flags); in reserve_highatomic_pageblock()
1928 struct zone *zone; in unreserve_highatomic_pageblock() local
1933 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, in unreserve_highatomic_pageblock()
1939 if (!force && zone->nr_reserved_highatomic <= in unreserve_highatomic_pageblock()
1943 spin_lock_irqsave(&zone->lock, flags); in unreserve_highatomic_pageblock()
1945 struct free_area *area = &(zone->free_area[order]); in unreserve_highatomic_pageblock()
1966 zone->nr_reserved_highatomic -= min( in unreserve_highatomic_pageblock()
1968 zone->nr_reserved_highatomic); in unreserve_highatomic_pageblock()
1981 ret = move_freepages_block(zone, page, ac->migratetype, in unreserve_highatomic_pageblock()
1984 spin_unlock_irqrestore(&zone->lock, flags); in unreserve_highatomic_pageblock()
1988 spin_unlock_irqrestore(&zone->lock, flags); in unreserve_highatomic_pageblock()
2005 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, in __rmqueue_fallback() argument
2030 area = &(zone->free_area[current_order]); in __rmqueue_fallback()
2055 area = &(zone->free_area[current_order]); in __rmqueue_fallback()
2071 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype, in __rmqueue_fallback()
2083 * Call me with the zone->lock already held.
2086 __rmqueue(struct zone *zone, unsigned int order, int migratetype, in __rmqueue() argument
2094 * allocating from CMA when over half of the zone's free memory in __rmqueue()
2098 zone_page_state(zone, NR_FREE_CMA_PAGES) > in __rmqueue()
2099 zone_page_state(zone, NR_FREE_PAGES) / 2) { in __rmqueue()
2100 page = __rmqueue_cma_fallback(zone, order); in __rmqueue()
2106 page = __rmqueue_smallest(zone, order, migratetype); in __rmqueue()
2109 page = __rmqueue_cma_fallback(zone, order); in __rmqueue()
2111 if (!page && __rmqueue_fallback(zone, order, migratetype, in __rmqueue()
2123 static int rmqueue_bulk(struct zone *zone, unsigned int order, in rmqueue_bulk() argument
2130 spin_lock_irqsave(&zone->lock, flags); in rmqueue_bulk()
2132 struct page *page = __rmqueue(zone, order, migratetype, in rmqueue_bulk()
2149 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, in rmqueue_bulk()
2153 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); in rmqueue_bulk()
2154 spin_unlock_irqrestore(&zone->lock, flags); in rmqueue_bulk()
2163 int decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp) in decay_pcp_high() argument
2185 free_pcppages_bulk(zone, to_drain, pcp, 0); in decay_pcp_high()
2199 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) in drain_zone_pages() argument
2207 free_pcppages_bulk(zone, to_drain, pcp, 0); in drain_zone_pages()
2214 * Drain pcplists of the indicated processor and zone.
2216 static void drain_pages_zone(unsigned int cpu, struct zone *zone) in drain_pages_zone() argument
2220 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in drain_pages_zone()
2223 free_pcppages_bulk(zone, pcp->count, pcp, 0); in drain_pages_zone()
2233 struct zone *zone; in drain_pages() local
2235 for_each_populated_zone(zone) { in drain_pages()
2236 drain_pages_zone(cpu, zone); in drain_pages()
2243 void drain_local_pages(struct zone *zone) in drain_local_pages() argument
2247 if (zone) in drain_local_pages()
2248 drain_pages_zone(cpu, zone); in drain_local_pages()
2263 static void __drain_all_pages(struct zone *zone, bool force_all_cpus) in __drain_all_pages() argument
2275 * a zone. Such callers are primarily CMA and memory hotplug and need in __drain_all_pages()
2279 if (!zone) in __drain_all_pages()
2292 struct zone *z; in __drain_all_pages()
2301 } else if (zone) { in __drain_all_pages()
2302 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in __drain_all_pages()
2322 if (zone) in __drain_all_pages()
2323 drain_pages_zone(cpu, zone); in __drain_all_pages()
2334 * When zone parameter is non-NULL, spill just the single zone's pages.
2336 void drain_all_pages(struct zone *zone) in drain_all_pages() argument
2338 __drain_all_pages(zone, false); in drain_all_pages()
2372 * freed pages to reduce zone lock contention. in nr_pcp_free()
2379 static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone, in nr_pcp_high() argument
2401 if (test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) { in nr_pcp_high()
2411 if (test_bit(ZONE_BELOW_HIGH, &zone->flags)) { in nr_pcp_high()
2427 static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, in free_unref_page_commit() argument
2464 high = nr_pcp_high(pcp, zone, batch, free_high); in free_unref_page_commit()
2466 free_pcppages_bulk(zone, nr_pcp_free(pcp, batch, high, free_high), in free_unref_page_commit()
2468 if (test_bit(ZONE_BELOW_HIGH, &zone->flags) && in free_unref_page_commit()
2469 zone_watermark_ok(zone, 0, high_wmark_pages(zone), in free_unref_page_commit()
2471 clear_bit(ZONE_BELOW_HIGH, &zone->flags); in free_unref_page_commit()
2482 struct zone *zone; in free_unref_page() local
2505 zone = page_zone(page); in free_unref_page()
2507 pcp = pcp_spin_trylock(zone->per_cpu_pageset); in free_unref_page()
2509 free_unref_page_commit(zone, pcp, page, pcpmigratetype, order); in free_unref_page()
2512 free_one_page(zone, page, pfn, order, migratetype, FPI_NONE); in free_unref_page()
2525 struct zone *locked_zone = NULL; in free_unref_page_list()
2550 struct zone *zone = page_zone(page); in free_unref_page_list() local
2556 * Either different zone requiring a different pcp lock or in free_unref_page_list()
2560 if (zone != locked_zone || batch_count == SWAP_CLUSTER_MAX) { in free_unref_page_list()
2573 pcp = pcp_spin_trylock(zone->per_cpu_pageset); in free_unref_page_list()
2576 free_one_page(zone, page, page_to_pfn(page), in free_unref_page_list()
2581 locked_zone = zone; in free_unref_page_list()
2592 free_unref_page_commit(zone, pcp, page, migratetype, 0); in free_unref_page_list()
2626 struct zone *zone = page_zone(page); in __isolate_free_page() local
2637 watermark = zone->_watermark[WMARK_MIN] + (1UL << order); in __isolate_free_page()
2638 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) in __isolate_free_page()
2641 __mod_zone_freepage_state(zone, -(1UL << order), mt); in __isolate_free_page()
2644 del_page_from_free_list(page, zone, order); in __isolate_free_page()
2678 struct zone *zone = page_zone(page); in __putback_isolated_page() local
2680 /* zone lock should be held when this function is called */ in __putback_isolated_page()
2681 lockdep_assert_held(&zone->lock); in __putback_isolated_page()
2684 __free_one_page(page, page_to_pfn(page), zone, order, mt, in __putback_isolated_page()
2691 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, in zone_statistics()
2715 struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, in rmqueue_buddy() argument
2724 spin_lock_irqsave(&zone->lock, flags); in rmqueue_buddy()
2726 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); in rmqueue_buddy()
2728 page = __rmqueue(zone, order, migratetype, alloc_flags); in rmqueue_buddy()
2737 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); in rmqueue_buddy()
2740 spin_unlock_irqrestore(&zone->lock, flags); in rmqueue_buddy()
2744 __mod_zone_freepage_state(zone, -(1 << order), in rmqueue_buddy()
2746 spin_unlock_irqrestore(&zone->lock, flags); in rmqueue_buddy()
2750 zone_statistics(preferred_zone, zone, 1); in rmqueue_buddy()
2755 static int nr_pcp_alloc(struct per_cpu_pages *pcp, struct zone *zone, int order) in nr_pcp_alloc() argument
2776 * zone. in nr_pcp_alloc()
2778 if (high_min != high_max && !test_bit(ZONE_BELOW_HIGH, &zone->flags)) in nr_pcp_alloc()
2807 struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, in __rmqueue_pcplist() argument
2817 int batch = nr_pcp_alloc(pcp, zone, order); in __rmqueue_pcplist()
2820 alloced = rmqueue_bulk(zone, order, in __rmqueue_pcplist()
2838 static struct page *rmqueue_pcplist(struct zone *preferred_zone, in rmqueue_pcplist()
2839 struct zone *zone, unsigned int order, in rmqueue_pcplist() argument
2849 pcp = pcp_spin_trylock(zone->per_cpu_pageset); in rmqueue_pcplist()
2862 page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list); in rmqueue_pcplist()
2867 zone_statistics(preferred_zone, zone, 1); in rmqueue_pcplist()
2873 * Allocate a page from the given zone.
2885 struct page *rmqueue(struct zone *preferred_zone, in rmqueue()
2886 struct zone *zone, unsigned int order, in rmqueue() argument
2899 page = rmqueue_pcplist(preferred_zone, zone, order, in rmqueue()
2905 page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags, in rmqueue()
2911 unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) { in rmqueue()
2912 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags); in rmqueue()
2913 wakeup_kswapd(zone, 0, 0, zone_idx(zone)); in rmqueue()
2916 VM_BUG_ON_PAGE(page && bad_range(zone, page), page); in rmqueue()
2926 static inline long __zone_watermark_unusable_free(struct zone *z, in __zone_watermark_unusable_free()
2954 * one free page of a suitable size. Checking now avoids taking the zone lock
2957 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, in __zone_watermark_ok()
3035 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, in zone_watermark_ok()
3042 static inline bool zone_watermark_fast(struct zone *z, unsigned int order, in zone_watermark_fast()
3087 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, in zone_watermark_ok_safe()
3102 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) in zone_allows_reclaim() argument
3104 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <= in zone_allows_reclaim()
3108 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) in zone_allows_reclaim() argument
3115 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3116 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3117 * premature use of a lower zone may cause lowmem pressure problems that
3118 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3123 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) in alloc_flags_nofragment() argument
3134 if (!zone) in alloc_flags_nofragment()
3137 if (zone_idx(zone) != ZONE_NORMAL) in alloc_flags_nofragment()
3142 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume in alloc_flags_nofragment()
3146 if (nr_online_nodes > 1 && !populated_zone(--zone)) in alloc_flags_nofragment()
3174 struct zone *zone; in get_page_from_freelist() local
3181 * Scan zonelist, looking for a zone with enough free. in get_page_from_freelist()
3186 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, in get_page_from_freelist()
3193 !__cpuset_zone_allowed(zone, gfp_mask)) in get_page_from_freelist()
3215 if (last_pgdat != zone->zone_pgdat) { in get_page_from_freelist()
3216 last_pgdat = zone->zone_pgdat; in get_page_from_freelist()
3217 last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat); in get_page_from_freelist()
3225 zone != ac->preferred_zoneref->zone) { in get_page_from_freelist()
3233 local_nid = zone_to_nid(ac->preferred_zoneref->zone); in get_page_from_freelist()
3234 if (zone_to_nid(zone) != local_nid) { in get_page_from_freelist()
3247 if (test_bit(ZONE_BELOW_HIGH, &zone->flags)) in get_page_from_freelist()
3250 mark = high_wmark_pages(zone); in get_page_from_freelist()
3251 if (zone_watermark_fast(zone, order, mark, in get_page_from_freelist()
3256 set_bit(ZONE_BELOW_HIGH, &zone->flags); in get_page_from_freelist()
3259 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK); in get_page_from_freelist()
3260 if (!zone_watermark_fast(zone, order, mark, in get_page_from_freelist()
3266 if (try_to_accept_memory(zone, order)) in get_page_from_freelist()
3272 * Watermark failed for this zone, but see if we can in get_page_from_freelist()
3273 * grow this zone if it contains deferred pages. in get_page_from_freelist()
3276 if (_deferred_grow_zone(zone, order)) in get_page_from_freelist()
3286 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone)) in get_page_from_freelist()
3289 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order); in get_page_from_freelist()
3299 if (zone_watermark_ok(zone, order, mark, in get_page_from_freelist()
3308 page = rmqueue(ac->preferred_zoneref->zone, zone, order, in get_page_from_freelist()
3318 reserve_highatomic_pageblock(page, zone); in get_page_from_freelist()
3323 if (try_to_accept_memory(zone, order)) in get_page_from_freelist()
3328 /* Try again if zone has deferred pages */ in get_page_from_freelist()
3330 if (_deferred_grow_zone(zone, order)) in get_page_from_freelist()
3534 * At least in one zone compaction wasn't deferred or skipped, so let's in __alloc_pages_direct_compact()
3548 struct zone *zone = page_zone(page); in __alloc_pages_direct_compact() local
3550 zone->compact_blockskip_flush = false; in __alloc_pages_direct_compact()
3551 compaction_defer_reset(zone, order, true); in __alloc_pages_direct_compact()
3648 struct zone *zone; in should_compact_retry() local
3660 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, in should_compact_retry()
3662 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone), in should_compact_retry()
3818 struct zone *zone; in wake_all_kswapds() local
3822 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx, in wake_all_kswapds()
3824 if (!managed_zone(zone)) in wake_all_kswapds()
3826 if (last_pgdat != zone->zone_pgdat) { in wake_all_kswapds()
3827 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx); in wake_all_kswapds()
3828 last_pgdat = zone->zone_pgdat; in wake_all_kswapds()
3939 struct zone *zone; in should_reclaim_retry() local
3963 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, in should_reclaim_retry()
3967 unsigned long min_wmark = min_wmark_pages(zone); in should_reclaim_retry()
3970 available = reclaimable = zone_reclaimable_pages(zone); in should_reclaim_retry()
3971 available += zone_page_state_snapshot(zone, NR_FREE_PAGES); in should_reclaim_retry()
3977 wmark = __zone_watermark_ok(zone, order, min_wmark, in should_reclaim_retry()
4079 if (!ac->preferred_zoneref->zone) in __alloc_pages_slowpath()
4084 * any suitable zone to satisfy the request - e.g. non-movable in __alloc_pages_slowpath()
4091 if (!z->zone) in __alloc_pages_slowpath()
4347 /* Dirty zone balancing only done in the fast path */ in prepare_alloc_pages()
4351 * The preferred zone is used for statistics but crucially it is in prepare_alloc_pages()
4388 struct zone *zone; in __alloc_pages_bulk() local
4439 /* Find an allowed local zone that meets the low watermark. */ in __alloc_pages_bulk()
4440 for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) { in __alloc_pages_bulk()
4444 !__cpuset_zone_allowed(zone, gfp)) { in __alloc_pages_bulk()
4448 if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone && in __alloc_pages_bulk()
4449 zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) { in __alloc_pages_bulk()
4453 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages; in __alloc_pages_bulk()
4454 if (zone_watermark_fast(zone, 0, mark, in __alloc_pages_bulk()
4465 if (unlikely(!zone)) in __alloc_pages_bulk()
4470 pcp = pcp_spin_trylock(zone->per_cpu_pageset); in __alloc_pages_bulk()
4484 page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags, in __alloc_pages_bulk()
4507 __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account); in __alloc_pages_bulk()
4508 zone_statistics(ac.preferred_zoneref->zone, zone, nr_account); in __alloc_pages_bulk()
4566 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp); in __alloc_pages()
4893 * @offset: The zone index of the highest zone
4896 * high watermark within all zones at or below a given zone index. For each
4897 * zone, the number of pages is calculated as:
4906 struct zone *zone; in nr_free_zone_pages() local
4913 for_each_zone_zonelist(zone, z, zonelist, offset) { in nr_free_zone_pages()
4914 unsigned long size = zone_managed_pages(zone); in nr_free_zone_pages()
4915 unsigned long high = high_wmark_pages(zone); in nr_free_zone_pages()
4938 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) in zoneref_set_zone() argument
4940 zoneref->zone = zone; in zoneref_set_zone()
4941 zoneref->zone_idx = zone_idx(zone); in zoneref_set_zone()
4945 * Builds allocation fallback zone lists.
4951 struct zone *zone; in build_zonerefs_node() local
4957 zone = pgdat->node_zones + zone_type; in build_zonerefs_node()
4958 if (populated_zone(zone)) { in build_zonerefs_node()
4959 zoneref_set_zone(zone, &zonerefs[nr_zones++]); in build_zonerefs_node()
5064 * This results in maximum locality--normal zone overflows into local
5065 * DMA zone, if any--but risks exhausting DMA zone.
5083 zonerefs->zone = NULL; in build_zonelists_in_node_order()
5098 zonerefs->zone = NULL; in build_thisnode_zonelists()
5103 * Build zonelists ordered by zone and nodes within zones.
5104 * This results in conserving DMA zone[s] until all Normal memory is
5106 * may still exist in local DMA zone.
5146 * I.e., first node id of first zone in arg node's generic zonelist.
5157 return zone_to_nid(z->zone); in local_memory_node()
5198 zonerefs->zone = NULL; in build_zonelists()
5217 * Other parts of the kernel may not check if the zone is available.
5270 * i.e., the node of the first zone in the generic zonelist. in __build_all_zonelists()
5295 * each zone will be allocated later when the per cpu in build_all_zonelists_init()
5333 * more accurate, but expensive to check per-zone. This check is in build_all_zonelists()
5347 pr_info("Policy zone: %s\n", zone_names[policy_zone]); in build_all_zonelists()
5351 static int zone_batchsize(struct zone *zone) in zone_batchsize() argument
5358 * of the zone or 1MB, whichever is smaller. The batch in zone_batchsize()
5360 * and zone lock contention. in zone_batchsize()
5362 batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); in zone_batchsize()
5400 static int zone_highsize(struct zone *zone, int batch, int cpu_online, in zone_highsize() argument
5410 * By default, the high value of the pcp is based on the zone in zone_highsize()
5414 total_pages = low_wmark_pages(zone); in zone_highsize()
5419 * zone. in zone_highsize()
5421 total_pages = zone_managed_pages(zone) / high_fraction; in zone_highsize()
5425 * Split the high value across all online CPUs local to the zone. Note in zone_highsize()
5432 nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online; in zone_highsize()
5497 static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high_min, in __zone_set_pageset_high_and_batch() argument
5504 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in __zone_set_pageset_high_and_batch()
5511 * zone based on the zone's size.
5513 static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) in zone_set_pageset_high_and_batch() argument
5517 new_batch = max(1, zone_batchsize(zone)); in zone_set_pageset_high_and_batch()
5519 new_high_min = zone_highsize(zone, new_batch, cpu_online, in zone_set_pageset_high_and_batch()
5527 new_high_min = zone_highsize(zone, new_batch, cpu_online, 0); in zone_set_pageset_high_and_batch()
5528 new_high_max = zone_highsize(zone, new_batch, cpu_online, in zone_set_pageset_high_and_batch()
5532 if (zone->pageset_high_min == new_high_min && in zone_set_pageset_high_and_batch()
5533 zone->pageset_high_max == new_high_max && in zone_set_pageset_high_and_batch()
5534 zone->pageset_batch == new_batch) in zone_set_pageset_high_and_batch()
5537 zone->pageset_high_min = new_high_min; in zone_set_pageset_high_and_batch()
5538 zone->pageset_high_max = new_high_max; in zone_set_pageset_high_and_batch()
5539 zone->pageset_batch = new_batch; in zone_set_pageset_high_and_batch()
5541 __zone_set_pageset_high_and_batch(zone, new_high_min, new_high_max, in zone_set_pageset_high_and_batch()
5545 void __meminit setup_zone_pageset(struct zone *zone) in setup_zone_pageset() argument
5551 zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat); in setup_zone_pageset()
5553 zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages); in setup_zone_pageset()
5558 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in setup_zone_pageset()
5559 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); in setup_zone_pageset()
5563 zone_set_pageset_high_and_batch(zone, 0); in setup_zone_pageset()
5567 * The zone indicated has a new number of managed_pages; batch sizes and percpu
5570 static void zone_pcp_update(struct zone *zone, int cpu_online) in zone_pcp_update() argument
5573 zone_set_pageset_high_and_batch(zone, cpu_online); in zone_pcp_update()
5577 static void zone_pcp_update_cacheinfo(struct zone *zone) in zone_pcp_update_cacheinfo() argument
5584 pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); in zone_pcp_update_cacheinfo()
5590 * This can reduce zone lock contention without hurting in zone_pcp_update_cacheinfo()
5604 struct zone *zone; in setup_pcp_cacheinfo() local
5606 for_each_populated_zone(zone) in setup_pcp_cacheinfo()
5607 zone_pcp_update_cacheinfo(zone); in setup_pcp_cacheinfo()
5617 struct zone *zone; in setup_per_cpu_pageset() local
5620 for_each_populated_zone(zone) in setup_per_cpu_pageset()
5621 setup_zone_pageset(zone); in setup_per_cpu_pageset()
5642 __meminit void zone_pcp_init(struct zone *zone) in zone_pcp_init() argument
5649 zone->per_cpu_pageset = &boot_pageset; in zone_pcp_init()
5650 zone->per_cpu_zonestats = &boot_zonestats; in zone_pcp_init()
5651 zone->pageset_high_min = BOOT_PAGESET_HIGH; in zone_pcp_init()
5652 zone->pageset_high_max = BOOT_PAGESET_HIGH; in zone_pcp_init()
5653 zone->pageset_batch = BOOT_PAGESET_BATCH; in zone_pcp_init()
5655 if (populated_zone(zone)) in zone_pcp_init()
5656 pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name, in zone_pcp_init()
5657 zone->present_pages, zone_batchsize(zone)); in zone_pcp_init()
5709 struct zone *zone; in page_alloc_cpu_dead() local
5732 for_each_populated_zone(zone) in page_alloc_cpu_dead()
5733 zone_pcp_update(zone, 0); in page_alloc_cpu_dead()
5740 struct zone *zone; in page_alloc_cpu_online() local
5742 for_each_populated_zone(zone) in page_alloc_cpu_online()
5743 zone_pcp_update(zone, 1); in page_alloc_cpu_online()
5773 struct zone *zone = pgdat->node_zones + i; in calculate_totalreserve_pages() local
5775 unsigned long managed_pages = zone_managed_pages(zone); in calculate_totalreserve_pages()
5777 /* Find valid and maximum lowmem_reserve in the zone */ in calculate_totalreserve_pages()
5779 if (zone->lowmem_reserve[j] > max) in calculate_totalreserve_pages()
5780 max = zone->lowmem_reserve[j]; in calculate_totalreserve_pages()
5784 max += high_wmark_pages(zone); in calculate_totalreserve_pages()
5799 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
5801 * pages are left in the zone after a successful __alloc_pages().
5810 struct zone *zone = &pgdat->node_zones[i]; in setup_per_zone_lowmem_reserve() local
5812 bool clear = !ratio || !zone_managed_pages(zone); in setup_per_zone_lowmem_reserve()
5816 struct zone *upper_zone = &pgdat->node_zones[j]; in setup_per_zone_lowmem_reserve()
5821 zone->lowmem_reserve[j] = 0; in setup_per_zone_lowmem_reserve()
5823 zone->lowmem_reserve[j] = managed_pages / ratio; in setup_per_zone_lowmem_reserve()
5836 struct zone *zone; in __setup_per_zone_wmarks() local
5840 for_each_zone(zone) { in __setup_per_zone_wmarks()
5841 if (!is_highmem(zone) && zone_idx(zone) != ZONE_MOVABLE) in __setup_per_zone_wmarks()
5842 lowmem_pages += zone_managed_pages(zone); in __setup_per_zone_wmarks()
5845 for_each_zone(zone) { in __setup_per_zone_wmarks()
5848 spin_lock_irqsave(&zone->lock, flags); in __setup_per_zone_wmarks()
5849 tmp = (u64)pages_min * zone_managed_pages(zone); in __setup_per_zone_wmarks()
5851 if (is_highmem(zone) || zone_idx(zone) == ZONE_MOVABLE) { in __setup_per_zone_wmarks()
5863 min_pages = zone_managed_pages(zone) / 1024; in __setup_per_zone_wmarks()
5865 zone->_watermark[WMARK_MIN] = min_pages; in __setup_per_zone_wmarks()
5868 * If it's a lowmem zone, reserve a number of pages in __setup_per_zone_wmarks()
5869 * proportionate to the zone's size. in __setup_per_zone_wmarks()
5871 zone->_watermark[WMARK_MIN] = tmp; in __setup_per_zone_wmarks()
5880 mult_frac(zone_managed_pages(zone), in __setup_per_zone_wmarks()
5883 zone->watermark_boost = 0; in __setup_per_zone_wmarks()
5884 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp; in __setup_per_zone_wmarks()
5885 zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp; in __setup_per_zone_wmarks()
5886 zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp; in __setup_per_zone_wmarks()
5888 spin_unlock_irqrestore(&zone->lock, flags); in __setup_per_zone_wmarks()
5899 * Ensures that the watermark[min,low,high] values for each zone are set
5904 struct zone *zone; in setup_per_zone_wmarks() local
5915 for_each_zone(zone) in setup_per_zone_wmarks()
5916 zone_pcp_update(zone, 0); in setup_per_zone_wmarks()
6017 struct zone *zone; in setup_min_unmapped_ratio() local
6022 for_each_zone(zone) in setup_min_unmapped_ratio()
6023 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) * in setup_min_unmapped_ratio()
6045 struct zone *zone; in setup_min_slab_ratio() local
6050 for_each_zone(zone) in setup_min_slab_ratio()
6051 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) * in setup_min_slab_ratio()
6077 * if in function of the boot time zone sizes.
6096 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each
6097 * cpu. It is the fraction of total pages in each zone that a hot per cpu
6103 struct zone *zone; in percpu_pagelist_high_fraction_sysctl_handler() local
6126 for_each_populated_zone(zone) in percpu_pagelist_high_fraction_sysctl_handler()
6127 zone_set_pageset_high_and_batch(zone, 0); in percpu_pagelist_high_fraction_sysctl_handler()
6224 /* [start, end) must belong to a single zone. */
6234 .nid = zone_to_nid(cc->zone), in __alloc_contig_migrate_range()
6258 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, in __alloc_contig_migrate_range()
6294 * belong to a single zone.
6314 .zone = page_zone(pfn_to_page(start)), in alloc_contig_range()
6348 drain_all_pages(cc.zone); in alloc_contig_range()
6378 * We don't have to hold zone->lock here because the pages are in alloc_contig_range()
6439 static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, in pfn_range_valid_contig()
6462 static bool zone_spans_last_pfn(const struct zone *zone, in zone_spans_last_pfn() argument
6467 return zone_spans_pfn(zone, last_pfn); in zone_spans_last_pfn()
6496 struct zone *zone; in alloc_contig_pages() local
6500 for_each_zone_zonelist_nodemask(zone, z, zonelist, in alloc_contig_pages()
6502 spin_lock_irqsave(&zone->lock, flags); in alloc_contig_pages()
6504 pfn = ALIGN(zone->zone_start_pfn, nr_pages); in alloc_contig_pages()
6505 while (zone_spans_last_pfn(zone, pfn, nr_pages)) { in alloc_contig_pages()
6506 if (pfn_range_valid_contig(zone, pfn, nr_pages)) { in alloc_contig_pages()
6508 * We release the zone lock here because in alloc_contig_pages()
6509 * alloc_contig_range() will also lock the zone in alloc_contig_pages()
6514 spin_unlock_irqrestore(&zone->lock, flags); in alloc_contig_pages()
6519 spin_lock_irqsave(&zone->lock, flags); in alloc_contig_pages()
6523 spin_unlock_irqrestore(&zone->lock, flags); in alloc_contig_pages()
6544 * Effectively disable pcplists for the zone by setting the high limit to 0
6551 void zone_pcp_disable(struct zone *zone) in zone_pcp_disable() argument
6554 __zone_set_pageset_high_and_batch(zone, 0, 0, 1); in zone_pcp_disable()
6555 __drain_all_pages(zone, true); in zone_pcp_disable()
6558 void zone_pcp_enable(struct zone *zone) in zone_pcp_enable() argument
6560 __zone_set_pageset_high_and_batch(zone, zone->pageset_high_min, in zone_pcp_enable()
6561 zone->pageset_high_max, zone->pageset_batch); in zone_pcp_enable()
6565 void zone_pcp_reset(struct zone *zone) in zone_pcp_reset() argument
6570 if (zone->per_cpu_pageset != &boot_pageset) { in zone_pcp_reset()
6572 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); in zone_pcp_reset()
6573 drain_zonestat(zone, pzstats); in zone_pcp_reset()
6575 free_percpu(zone->per_cpu_pageset); in zone_pcp_reset()
6576 zone->per_cpu_pageset = &boot_pageset; in zone_pcp_reset()
6577 if (zone->per_cpu_zonestats != &boot_zonestats) { in zone_pcp_reset()
6578 free_percpu(zone->per_cpu_zonestats); in zone_pcp_reset()
6579 zone->per_cpu_zonestats = &boot_zonestats; in zone_pcp_reset()
6586 * All pages in the range must be in a single zone, must not contain holes,
6593 struct zone *zone; in __offline_isolated_pages() local
6598 zone = page_zone(pfn_to_page(pfn)); in __offline_isolated_pages()
6599 spin_lock_irqsave(&zone->lock, flags); in __offline_isolated_pages()
6624 del_page_from_free_list(page, zone, order); in __offline_isolated_pages()
6627 spin_unlock_irqrestore(&zone->lock, flags); in __offline_isolated_pages()
6632 * This function returns a stable result only if called under zone lock.
6656 static void break_down_buddy_pages(struct zone *zone, struct page *page, in break_down_buddy_pages() argument
6674 if (set_page_guard(zone, current_buddy, high, migratetype)) in break_down_buddy_pages()
6677 add_to_free_list(current_buddy, zone, high, migratetype); in break_down_buddy_pages()
6687 struct zone *zone = page_zone(page); in take_page_off_buddy() local
6693 spin_lock_irqsave(&zone->lock, flags); in take_page_off_buddy()
6703 del_page_from_free_list(page_head, zone, page_order); in take_page_off_buddy()
6704 break_down_buddy_pages(zone, page_head, page, 0, in take_page_off_buddy()
6708 __mod_zone_freepage_state(zone, -1, migratetype); in take_page_off_buddy()
6715 spin_unlock_irqrestore(&zone->lock, flags); in take_page_off_buddy()
6724 struct zone *zone = page_zone(page); in put_page_back_buddy() local
6730 spin_lock_irqsave(&zone->lock, flags); in put_page_back_buddy()
6733 __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE); in put_page_back_buddy()
6738 spin_unlock_irqrestore(&zone->lock, flags); in put_page_back_buddy()
6750 struct zone *zone = &pgdat->node_zones[ZONE_DMA]; in has_managed_dma() local
6752 if (managed_zone(zone)) in has_managed_dma()
6795 static bool try_to_accept_memory_one(struct zone *zone) in try_to_accept_memory_one() argument
6801 if (list_empty(&zone->unaccepted_pages)) in try_to_accept_memory_one()
6804 spin_lock_irqsave(&zone->lock, flags); in try_to_accept_memory_one()
6805 page = list_first_entry_or_null(&zone->unaccepted_pages, in try_to_accept_memory_one()
6808 spin_unlock_irqrestore(&zone->lock, flags); in try_to_accept_memory_one()
6813 last = list_empty(&zone->unaccepted_pages); in try_to_accept_memory_one()
6815 __mod_zone_freepage_state(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); in try_to_accept_memory_one()
6816 __mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES); in try_to_accept_memory_one()
6817 spin_unlock_irqrestore(&zone->lock, flags); in try_to_accept_memory_one()
6829 static bool try_to_accept_memory(struct zone *zone, unsigned int order) in try_to_accept_memory() argument
6835 to_accept = high_wmark_pages(zone) - in try_to_accept_memory()
6836 (zone_page_state(zone, NR_FREE_PAGES) - in try_to_accept_memory()
6837 __zone_watermark_unusable_free(zone, order, 0)); in try_to_accept_memory()
6841 if (!try_to_accept_memory_one(zone)) in try_to_accept_memory()
6857 struct zone *zone = page_zone(page); in __free_unaccepted() local
6864 spin_lock_irqsave(&zone->lock, flags); in __free_unaccepted()
6865 first = list_empty(&zone->unaccepted_pages); in __free_unaccepted()
6866 list_add_tail(&page->lru, &zone->unaccepted_pages); in __free_unaccepted()
6867 __mod_zone_freepage_state(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE); in __free_unaccepted()
6868 __mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES); in __free_unaccepted()
6869 spin_unlock_irqrestore(&zone->lock, flags); in __free_unaccepted()
6888 static bool try_to_accept_memory(struct zone *zone, unsigned int order) in try_to_accept_memory() argument