1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_VMSTAT_H
3 #define _LINUX_VMSTAT_H
4
5 #include <linux/types.h>
6 #include <linux/percpu.h>
7 #include <linux/mmzone.h>
8 #include <linux/vm_event_item.h>
9 #include <linux/atomic.h>
10 #include <linux/static_key.h>
11 #include <linux/mmdebug.h>
12
13 #ifdef CONFIG_NUMA
14 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
15 #endif
16
17 struct reclaim_stat {
18 unsigned nr_dirty;
19 unsigned nr_unqueued_dirty;
20 unsigned nr_congested;
21 unsigned nr_writeback;
22 unsigned nr_immediate;
23 unsigned nr_pageout;
24 unsigned nr_activate[ANON_AND_FILE];
25 unsigned nr_ref_keep;
26 unsigned nr_unmap_fail;
27 unsigned nr_lazyfree_fail;
28 unsigned nr_demoted;
29 };
30
31 /* Stat data for system wide items */
32 enum vm_stat_item {
33 NR_DIRTY_THRESHOLD,
34 NR_DIRTY_BG_THRESHOLD,
35 NR_MEMMAP_PAGES, /* page metadata allocated through buddy allocator */
36 NR_MEMMAP_BOOT_PAGES, /* page metadata allocated through boot allocator */
37 NR_VM_STAT_ITEMS,
38 };
39
40 #ifdef CONFIG_VM_EVENT_COUNTERS
41 /*
42 * Light weight per cpu counter implementation.
43 *
44 * Counters should only be incremented and no critical kernel component
45 * should rely on the counter values.
46 *
47 * Counters are handled completely inline. On many platforms the code
48 * generated will simply be the increment of a global address.
49 */
50
51 struct vm_event_state {
52 unsigned long event[NR_VM_EVENT_ITEMS];
53 };
54
55 DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
56
57 /*
58 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
59 * local_irq_disable overhead.
60 */
__count_vm_event(enum vm_event_item item)61 static inline void __count_vm_event(enum vm_event_item item)
62 {
63 raw_cpu_inc(vm_event_states.event[item]);
64 }
65
count_vm_event(enum vm_event_item item)66 static inline void count_vm_event(enum vm_event_item item)
67 {
68 this_cpu_inc(vm_event_states.event[item]);
69 }
70
__count_vm_events(enum vm_event_item item,long delta)71 static inline void __count_vm_events(enum vm_event_item item, long delta)
72 {
73 raw_cpu_add(vm_event_states.event[item], delta);
74 }
75
count_vm_events(enum vm_event_item item,long delta)76 static inline void count_vm_events(enum vm_event_item item, long delta)
77 {
78 this_cpu_add(vm_event_states.event[item], delta);
79 }
80
81 extern void all_vm_events(unsigned long *);
82
83 extern void vm_events_fold_cpu(int cpu);
84
85 #else
86
87 /* Disable counters */
count_vm_event(enum vm_event_item item)88 static inline void count_vm_event(enum vm_event_item item)
89 {
90 }
count_vm_events(enum vm_event_item item,long delta)91 static inline void count_vm_events(enum vm_event_item item, long delta)
92 {
93 }
__count_vm_event(enum vm_event_item item)94 static inline void __count_vm_event(enum vm_event_item item)
95 {
96 }
__count_vm_events(enum vm_event_item item,long delta)97 static inline void __count_vm_events(enum vm_event_item item, long delta)
98 {
99 }
all_vm_events(unsigned long * ret)100 static inline void all_vm_events(unsigned long *ret)
101 {
102 }
vm_events_fold_cpu(int cpu)103 static inline void vm_events_fold_cpu(int cpu)
104 {
105 }
106
107 #endif /* CONFIG_VM_EVENT_COUNTERS */
108
109 #ifdef CONFIG_NUMA_BALANCING
110 #define count_vm_numa_event(x) count_vm_event(x)
111 #define count_vm_numa_events(x, y) count_vm_events(x, y)
112 #else
113 #define count_vm_numa_event(x) do {} while (0)
114 #define count_vm_numa_events(x, y) do { (void)(y); } while (0)
115 #endif /* CONFIG_NUMA_BALANCING */
116
117 #ifdef CONFIG_DEBUG_TLBFLUSH
118 #define count_vm_tlb_event(x) count_vm_event(x)
119 #define count_vm_tlb_events(x, y) count_vm_events(x, y)
120 #else
121 #define count_vm_tlb_event(x) do {} while (0)
122 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
123 #endif
124
125 #ifdef CONFIG_PER_VMA_LOCK_STATS
126 #define count_vm_vma_lock_event(x) count_vm_event(x)
127 #else
128 #define count_vm_vma_lock_event(x) do {} while (0)
129 #endif
130
131 #define __count_zid_vm_events(item, zid, delta) \
132 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
133
134 /*
135 * Zone and node-based page accounting with per cpu differentials.
136 */
137 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
138 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
139 extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
140
141 #ifdef CONFIG_NUMA
zone_numa_event_add(long x,struct zone * zone,enum numa_stat_item item)142 static inline void zone_numa_event_add(long x, struct zone *zone,
143 enum numa_stat_item item)
144 {
145 atomic_long_add(x, &zone->vm_numa_event[item]);
146 atomic_long_add(x, &vm_numa_event[item]);
147 }
148
zone_numa_event_state(struct zone * zone,enum numa_stat_item item)149 static inline unsigned long zone_numa_event_state(struct zone *zone,
150 enum numa_stat_item item)
151 {
152 return atomic_long_read(&zone->vm_numa_event[item]);
153 }
154
155 static inline unsigned long
global_numa_event_state(enum numa_stat_item item)156 global_numa_event_state(enum numa_stat_item item)
157 {
158 return atomic_long_read(&vm_numa_event[item]);
159 }
160 #endif /* CONFIG_NUMA */
161
zone_page_state_add(long x,struct zone * zone,enum zone_stat_item item)162 static inline void zone_page_state_add(long x, struct zone *zone,
163 enum zone_stat_item item)
164 {
165 atomic_long_add(x, &zone->vm_stat[item]);
166 atomic_long_add(x, &vm_zone_stat[item]);
167 }
168
node_page_state_add(long x,struct pglist_data * pgdat,enum node_stat_item item)169 static inline void node_page_state_add(long x, struct pglist_data *pgdat,
170 enum node_stat_item item)
171 {
172 atomic_long_add(x, &pgdat->vm_stat[item]);
173 atomic_long_add(x, &vm_node_stat[item]);
174 }
175
global_zone_page_state(enum zone_stat_item item)176 static inline unsigned long global_zone_page_state(enum zone_stat_item item)
177 {
178 long x = atomic_long_read(&vm_zone_stat[item]);
179 #ifdef CONFIG_SMP
180 if (x < 0)
181 x = 0;
182 #endif
183 return x;
184 }
185
186 static inline
global_node_page_state_pages(enum node_stat_item item)187 unsigned long global_node_page_state_pages(enum node_stat_item item)
188 {
189 long x = atomic_long_read(&vm_node_stat[item]);
190 #ifdef CONFIG_SMP
191 if (x < 0)
192 x = 0;
193 #endif
194 return x;
195 }
196
global_node_page_state(enum node_stat_item item)197 static inline unsigned long global_node_page_state(enum node_stat_item item)
198 {
199 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
200
201 return global_node_page_state_pages(item);
202 }
203
zone_page_state(struct zone * zone,enum zone_stat_item item)204 static inline unsigned long zone_page_state(struct zone *zone,
205 enum zone_stat_item item)
206 {
207 long x = atomic_long_read(&zone->vm_stat[item]);
208 #ifdef CONFIG_SMP
209 if (x < 0)
210 x = 0;
211 #endif
212 return x;
213 }
214
215 /*
216 * More accurate version that also considers the currently pending
217 * deltas. For that we need to loop over all cpus to find the current
218 * deltas. There is no synchronization so the result cannot be
219 * exactly accurate either.
220 */
zone_page_state_snapshot(struct zone * zone,enum zone_stat_item item)221 static inline unsigned long zone_page_state_snapshot(struct zone *zone,
222 enum zone_stat_item item)
223 {
224 long x = atomic_long_read(&zone->vm_stat[item]);
225
226 #ifdef CONFIG_SMP
227 int cpu;
228 for_each_online_cpu(cpu)
229 x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];
230
231 if (x < 0)
232 x = 0;
233 #endif
234 return x;
235 }
236
237 #ifdef CONFIG_NUMA
238 /* See __count_vm_event comment on why raw_cpu_inc is used. */
239 static inline void
__count_numa_event(struct zone * zone,enum numa_stat_item item)240 __count_numa_event(struct zone *zone, enum numa_stat_item item)
241 {
242 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
243
244 raw_cpu_inc(pzstats->vm_numa_event[item]);
245 }
246
247 static inline void
__count_numa_events(struct zone * zone,enum numa_stat_item item,long delta)248 __count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
249 {
250 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
251
252 raw_cpu_add(pzstats->vm_numa_event[item], delta);
253 }
254
255 extern unsigned long sum_zone_node_page_state(int node,
256 enum zone_stat_item item);
257 extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
258 extern unsigned long node_page_state(struct pglist_data *pgdat,
259 enum node_stat_item item);
260 extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
261 enum node_stat_item item);
262 extern void fold_vm_numa_events(void);
263 #else
264 #define sum_zone_node_page_state(node, item) global_zone_page_state(item)
265 #define node_page_state(node, item) global_node_page_state(item)
266 #define node_page_state_pages(node, item) global_node_page_state_pages(item)
fold_vm_numa_events(void)267 static inline void fold_vm_numa_events(void)
268 {
269 }
270 #endif /* CONFIG_NUMA */
271
272 #ifdef CONFIG_SMP
273 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
274 void __inc_zone_page_state(struct page *, enum zone_stat_item);
275 void __dec_zone_page_state(struct page *, enum zone_stat_item);
276
277 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
278 void __inc_node_page_state(struct page *, enum node_stat_item);
279 void __dec_node_page_state(struct page *, enum node_stat_item);
280
281 void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
282 void inc_zone_page_state(struct page *, enum zone_stat_item);
283 void dec_zone_page_state(struct page *, enum zone_stat_item);
284
285 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
286 void inc_node_page_state(struct page *, enum node_stat_item);
287 void dec_node_page_state(struct page *, enum node_stat_item);
288
289 extern void __inc_zone_state(struct zone *, enum zone_stat_item);
290 extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
291 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
292 extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
293
294 void quiet_vmstat(void);
295 void cpu_vm_stats_fold(int cpu);
296 void refresh_zone_stat_thresholds(void);
297
298 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *);
299
300 int calculate_pressure_threshold(struct zone *zone);
301 int calculate_normal_threshold(struct zone *zone);
302 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
303 int (*calculate_pressure)(struct zone *));
304 void vmstat_flush_workqueue(void);
305 #else /* CONFIG_SMP */
306
307 /*
308 * We do not maintain differentials in a single processor configuration.
309 * The functions directly modify the zone and global counters.
310 */
__mod_zone_page_state(struct zone * zone,enum zone_stat_item item,long delta)311 static inline void __mod_zone_page_state(struct zone *zone,
312 enum zone_stat_item item, long delta)
313 {
314 zone_page_state_add(delta, zone, item);
315 }
316
__mod_node_page_state(struct pglist_data * pgdat,enum node_stat_item item,int delta)317 static inline void __mod_node_page_state(struct pglist_data *pgdat,
318 enum node_stat_item item, int delta)
319 {
320 if (vmstat_item_in_bytes(item)) {
321 /*
322 * Only cgroups use subpage accounting right now; at
323 * the global level, these items still change in
324 * multiples of whole pages. Store them as pages
325 * internally to keep the per-cpu counters compact.
326 */
327 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
328 delta >>= PAGE_SHIFT;
329 }
330
331 node_page_state_add(delta, pgdat, item);
332 }
333
__inc_zone_state(struct zone * zone,enum zone_stat_item item)334 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
335 {
336 atomic_long_inc(&zone->vm_stat[item]);
337 atomic_long_inc(&vm_zone_stat[item]);
338 }
339
__inc_node_state(struct pglist_data * pgdat,enum node_stat_item item)340 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
341 {
342 atomic_long_inc(&pgdat->vm_stat[item]);
343 atomic_long_inc(&vm_node_stat[item]);
344 }
345
__dec_zone_state(struct zone * zone,enum zone_stat_item item)346 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
347 {
348 atomic_long_dec(&zone->vm_stat[item]);
349 atomic_long_dec(&vm_zone_stat[item]);
350 }
351
__dec_node_state(struct pglist_data * pgdat,enum node_stat_item item)352 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
353 {
354 atomic_long_dec(&pgdat->vm_stat[item]);
355 atomic_long_dec(&vm_node_stat[item]);
356 }
357
__inc_zone_page_state(struct page * page,enum zone_stat_item item)358 static inline void __inc_zone_page_state(struct page *page,
359 enum zone_stat_item item)
360 {
361 __inc_zone_state(page_zone(page), item);
362 }
363
__inc_node_page_state(struct page * page,enum node_stat_item item)364 static inline void __inc_node_page_state(struct page *page,
365 enum node_stat_item item)
366 {
367 __inc_node_state(page_pgdat(page), item);
368 }
369
370
__dec_zone_page_state(struct page * page,enum zone_stat_item item)371 static inline void __dec_zone_page_state(struct page *page,
372 enum zone_stat_item item)
373 {
374 __dec_zone_state(page_zone(page), item);
375 }
376
__dec_node_page_state(struct page * page,enum node_stat_item item)377 static inline void __dec_node_page_state(struct page *page,
378 enum node_stat_item item)
379 {
380 __dec_node_state(page_pgdat(page), item);
381 }
382
383
384 /*
385 * We only use atomic operations to update counters. So there is no need to
386 * disable interrupts.
387 */
388 #define inc_zone_page_state __inc_zone_page_state
389 #define dec_zone_page_state __dec_zone_page_state
390 #define mod_zone_page_state __mod_zone_page_state
391
392 #define inc_node_page_state __inc_node_page_state
393 #define dec_node_page_state __dec_node_page_state
394 #define mod_node_page_state __mod_node_page_state
395
396 #define set_pgdat_percpu_threshold(pgdat, callback) { }
397
refresh_zone_stat_thresholds(void)398 static inline void refresh_zone_stat_thresholds(void) { }
cpu_vm_stats_fold(int cpu)399 static inline void cpu_vm_stats_fold(int cpu) { }
quiet_vmstat(void)400 static inline void quiet_vmstat(void) { }
vmstat_flush_workqueue(void)401 static inline void vmstat_flush_workqueue(void) { }
402
drain_zonestat(struct zone * zone,struct per_cpu_zonestat * pzstats)403 static inline void drain_zonestat(struct zone *zone,
404 struct per_cpu_zonestat *pzstats) { }
405 #endif /* CONFIG_SMP */
406
__zone_stat_mod_folio(struct folio * folio,enum zone_stat_item item,long nr)407 static inline void __zone_stat_mod_folio(struct folio *folio,
408 enum zone_stat_item item, long nr)
409 {
410 __mod_zone_page_state(folio_zone(folio), item, nr);
411 }
412
__zone_stat_add_folio(struct folio * folio,enum zone_stat_item item)413 static inline void __zone_stat_add_folio(struct folio *folio,
414 enum zone_stat_item item)
415 {
416 __mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
417 }
418
__zone_stat_sub_folio(struct folio * folio,enum zone_stat_item item)419 static inline void __zone_stat_sub_folio(struct folio *folio,
420 enum zone_stat_item item)
421 {
422 __mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
423 }
424
zone_stat_mod_folio(struct folio * folio,enum zone_stat_item item,long nr)425 static inline void zone_stat_mod_folio(struct folio *folio,
426 enum zone_stat_item item, long nr)
427 {
428 mod_zone_page_state(folio_zone(folio), item, nr);
429 }
430
zone_stat_add_folio(struct folio * folio,enum zone_stat_item item)431 static inline void zone_stat_add_folio(struct folio *folio,
432 enum zone_stat_item item)
433 {
434 mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
435 }
436
zone_stat_sub_folio(struct folio * folio,enum zone_stat_item item)437 static inline void zone_stat_sub_folio(struct folio *folio,
438 enum zone_stat_item item)
439 {
440 mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
441 }
442
__node_stat_mod_folio(struct folio * folio,enum node_stat_item item,long nr)443 static inline void __node_stat_mod_folio(struct folio *folio,
444 enum node_stat_item item, long nr)
445 {
446 __mod_node_page_state(folio_pgdat(folio), item, nr);
447 }
448
__node_stat_add_folio(struct folio * folio,enum node_stat_item item)449 static inline void __node_stat_add_folio(struct folio *folio,
450 enum node_stat_item item)
451 {
452 __mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
453 }
454
__node_stat_sub_folio(struct folio * folio,enum node_stat_item item)455 static inline void __node_stat_sub_folio(struct folio *folio,
456 enum node_stat_item item)
457 {
458 __mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
459 }
460
node_stat_mod_folio(struct folio * folio,enum node_stat_item item,long nr)461 static inline void node_stat_mod_folio(struct folio *folio,
462 enum node_stat_item item, long nr)
463 {
464 mod_node_page_state(folio_pgdat(folio), item, nr);
465 }
466
node_stat_add_folio(struct folio * folio,enum node_stat_item item)467 static inline void node_stat_add_folio(struct folio *folio,
468 enum node_stat_item item)
469 {
470 mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
471 }
472
node_stat_sub_folio(struct folio * folio,enum node_stat_item item)473 static inline void node_stat_sub_folio(struct folio *folio,
474 enum node_stat_item item)
475 {
476 mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
477 }
478
479 extern const char * const vmstat_text[];
480
zone_stat_name(enum zone_stat_item item)481 static inline const char *zone_stat_name(enum zone_stat_item item)
482 {
483 return vmstat_text[item];
484 }
485
486 #ifdef CONFIG_NUMA
numa_stat_name(enum numa_stat_item item)487 static inline const char *numa_stat_name(enum numa_stat_item item)
488 {
489 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
490 item];
491 }
492 #endif /* CONFIG_NUMA */
493
node_stat_name(enum node_stat_item item)494 static inline const char *node_stat_name(enum node_stat_item item)
495 {
496 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
497 NR_VM_NUMA_EVENT_ITEMS +
498 item];
499 }
500
lru_list_name(enum lru_list lru)501 static inline const char *lru_list_name(enum lru_list lru)
502 {
503 return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
504 }
505
506 #if defined(CONFIG_VM_EVENT_COUNTERS)
vm_event_name(enum vm_event_item item)507 static inline const char *vm_event_name(enum vm_event_item item)
508 {
509 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
510 NR_VM_NUMA_EVENT_ITEMS +
511 NR_VM_NODE_STAT_ITEMS +
512 NR_VM_STAT_ITEMS +
513 item];
514 }
515 #endif /* CONFIG_VM_EVENT_COUNTERS */
516
517 #ifdef CONFIG_MEMCG
518
519 void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
520 int val);
521
522 void lruvec_stat_mod_folio(struct folio *folio,
523 enum node_stat_item idx, int val);
524
mod_lruvec_page_state(struct page * page,enum node_stat_item idx,int val)525 static inline void mod_lruvec_page_state(struct page *page,
526 enum node_stat_item idx, int val)
527 {
528 lruvec_stat_mod_folio(page_folio(page), idx, val);
529 }
530
531 #else
532
mod_lruvec_state(struct lruvec * lruvec,enum node_stat_item idx,int val)533 static inline void mod_lruvec_state(struct lruvec *lruvec,
534 enum node_stat_item idx, int val)
535 {
536 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
537 }
538
lruvec_stat_mod_folio(struct folio * folio,enum node_stat_item idx,int val)539 static inline void lruvec_stat_mod_folio(struct folio *folio,
540 enum node_stat_item idx, int val)
541 {
542 mod_node_page_state(folio_pgdat(folio), idx, val);
543 }
544
mod_lruvec_page_state(struct page * page,enum node_stat_item idx,int val)545 static inline void mod_lruvec_page_state(struct page *page,
546 enum node_stat_item idx, int val)
547 {
548 mod_node_page_state(page_pgdat(page), idx, val);
549 }
550
551 #endif /* CONFIG_MEMCG */
552
lruvec_stat_add_folio(struct folio * folio,enum node_stat_item idx)553 static inline void lruvec_stat_add_folio(struct folio *folio,
554 enum node_stat_item idx)
555 {
556 lruvec_stat_mod_folio(folio, idx, folio_nr_pages(folio));
557 }
558
lruvec_stat_sub_folio(struct folio * folio,enum node_stat_item idx)559 static inline void lruvec_stat_sub_folio(struct folio *folio,
560 enum node_stat_item idx)
561 {
562 lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio));
563 }
564
565 void memmap_boot_pages_add(long delta);
566 void memmap_pages_add(long delta);
567 #endif /* _LINUX_VMSTAT_H */
568