projects/linux·blob master · refs

582 lines · 15.9 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_VMSTAT_H
3#define _LINUX_VMSTAT_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7#include <linux/mmzone.h>
8#include <linux/vm_event_item.h>
9#include <linux/atomic.h>
10#include <linux/static_key.h>
11#include <linux/mmdebug.h>
12
13#ifdef CONFIG_NUMA
14DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
15#endif
16
17struct reclaim_stat {
18 unsigned nr_dirty;
19 unsigned nr_unqueued_dirty;
20 unsigned nr_congested;
21 unsigned nr_writeback;
22 unsigned nr_immediate;
23 unsigned nr_activate[ANON_AND_FILE];
24 unsigned nr_ref_keep;
25 unsigned nr_unmap_fail;
26 unsigned nr_lazyfree_fail;
27 unsigned nr_demoted;
28};
29
30/* Stat data for system wide items */
31enum vm_stat_item {
32 NR_DIRTY_THRESHOLD,
33 NR_DIRTY_BG_THRESHOLD,
34 NR_MEMMAP_PAGES, /* page metadata allocated through buddy allocator */
35 NR_MEMMAP_BOOT_PAGES, /* page metadata allocated through boot allocator */
36 NR_VM_STAT_ITEMS,
37};
38
39#ifdef CONFIG_VM_EVENT_COUNTERS
40/*
41 * Light weight per cpu counter implementation.
42 *
43 * Counters should only be incremented and no critical kernel component
44 * should rely on the counter values.
45 *
46 * Counters are handled completely inline. On many platforms the code
47 * generated will simply be the increment of a global address.
48 */
49
50struct vm_event_state {
51 unsigned long event[NR_VM_EVENT_ITEMS];
52};
53
54DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
55
56/*
57 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
58 * local_irq_disable overhead.
59 */
60static inline void __count_vm_event(enum vm_event_item item)
61{
62 raw_cpu_inc(vm_event_states.event[item]);
63}
64
65static inline void count_vm_event(enum vm_event_item item)
66{
67 this_cpu_inc(vm_event_states.event[item]);
68}
69
70static inline void __count_vm_events(enum vm_event_item item, long delta)
71{
72 raw_cpu_add(vm_event_states.event[item], delta);
73}
74
75static inline void count_vm_events(enum vm_event_item item, long delta)
76{
77 this_cpu_add(vm_event_states.event[item], delta);
78}
79
80extern void all_vm_events(unsigned long *);
81
82extern void vm_events_fold_cpu(int cpu);
83
84#else
85
86/* Disable counters */
87static inline void count_vm_event(enum vm_event_item item)
88{
89}
90static inline void count_vm_events(enum vm_event_item item, long delta)
91{
92}
93static inline void __count_vm_event(enum vm_event_item item)
94{
95}
96static inline void __count_vm_events(enum vm_event_item item, long delta)
97{
98}
99static inline void all_vm_events(unsigned long *ret)
100{
101}
102static inline void vm_events_fold_cpu(int cpu)
103{
104}
105
106#endif /* CONFIG_VM_EVENT_COUNTERS */
107
108#ifdef CONFIG_NUMA_BALANCING
109#define count_vm_numa_event(x) count_vm_event(x)
110#define count_vm_numa_events(x, y) count_vm_events(x, y)
111#else
112#define count_vm_numa_event(x) do {} while (0)
113#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
114#endif /* CONFIG_NUMA_BALANCING */
115
116#ifdef CONFIG_DEBUG_TLBFLUSH
117#define count_vm_tlb_event(x) count_vm_event(x)
118#define count_vm_tlb_events(x, y) count_vm_events(x, y)
119#else
120#define count_vm_tlb_event(x) do {} while (0)
121#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
122#endif
123
124#ifdef CONFIG_PER_VMA_LOCK_STATS
125#define count_vm_vma_lock_event(x) count_vm_event(x)
126#else
127#define count_vm_vma_lock_event(x) do {} while (0)
128#endif
129
130#define __count_zid_vm_events(item, zid, delta) \
131 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
132
133/*
134 * Zone and node-based page accounting with per cpu differentials.
135 */
136extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
137extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
138extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
139
140#ifdef CONFIG_NUMA
141static inline void zone_numa_event_add(long x, struct zone *zone,
142 enum numa_stat_item item)
143{
144 atomic_long_add(x, &zone->vm_numa_event[item]);
145 atomic_long_add(x, &vm_numa_event[item]);
146}
147
148static inline unsigned long zone_numa_event_state(struct zone *zone,
149 enum numa_stat_item item)
150{
151 return atomic_long_read(&zone->vm_numa_event[item]);
152}
153
154static inline unsigned long
155global_numa_event_state(enum numa_stat_item item)
156{
157 return atomic_long_read(&vm_numa_event[item]);
158}
159#endif /* CONFIG_NUMA */
160
161static inline void zone_page_state_add(long x, struct zone *zone,
162 enum zone_stat_item item)
163{
164 atomic_long_add(x, &zone->vm_stat[item]);
165 atomic_long_add(x, &vm_zone_stat[item]);
166}
167
168static inline void node_page_state_add(long x, struct pglist_data *pgdat,
169 enum node_stat_item item)
170{
171 atomic_long_add(x, &pgdat->vm_stat[item]);
172 atomic_long_add(x, &vm_node_stat[item]);
173}
174
175static inline unsigned long global_zone_page_state(enum zone_stat_item item)
176{
177 long x = atomic_long_read(&vm_zone_stat[item]);
178#ifdef CONFIG_SMP
179 if (x < 0)
180 x = 0;
181#endif
182 return x;
183}
184
185static inline
186unsigned long global_node_page_state_pages(enum node_stat_item item)
187{
188 long x = atomic_long_read(&vm_node_stat[item]);
189#ifdef CONFIG_SMP
190 if (x < 0)
191 x = 0;
192#endif
193 return x;
194}
195
196/*
197 * Non-clamping variant of global_node_page_state() intended for callers that
198 * snapshot a monotonically-incremented counter and subtract two samples.
199 * Returns the raw wrapping value so that unsigned modular subtraction stays
200 * correct across a signed-long overflow (a real hazard on 32-bit) that the
201 * clamp in global_node_page_state() would otherwise turn into a huge spurious
202 * delta. Do NOT use for non-monotonic page-count reads.
203 */
204static inline unsigned long global_node_page_state_monotonic(enum node_stat_item item)
205{
206 return (unsigned long)atomic_long_read(&vm_node_stat[item]);
207}
208
209static inline unsigned long global_node_page_state(enum node_stat_item item)
210{
211 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
212
213 return global_node_page_state_pages(item);
214}
215
216static inline unsigned long zone_page_state(struct zone *zone,
217 enum zone_stat_item item)
218{
219 long x = atomic_long_read(&zone->vm_stat[item]);
220#ifdef CONFIG_SMP
221 if (x < 0)
222 x = 0;
223#endif
224 return x;
225}
226
227/*
228 * More accurate version that also considers the currently pending
229 * deltas. For that we need to loop over all cpus to find the current
230 * deltas. There is no synchronization so the result cannot be
231 * exactly accurate either.
232 */
233static inline unsigned long zone_page_state_snapshot(struct zone *zone,
234 enum zone_stat_item item)
235{
236 long x = atomic_long_read(&zone->vm_stat[item]);
237
238#ifdef CONFIG_SMP
239 int cpu;
240 for_each_online_cpu(cpu)
241 x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];
242
243 if (x < 0)
244 x = 0;
245#endif
246 return x;
247}
248
249#ifdef CONFIG_NUMA
250/* See __count_vm_event comment on why raw_cpu_inc is used. */
251static inline void
252__count_numa_event(struct zone *zone, enum numa_stat_item item)
253{
254 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
255
256 raw_cpu_inc(pzstats->vm_numa_event[item]);
257}
258
259static inline void
260__count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
261{
262 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
263
264 raw_cpu_add(pzstats->vm_numa_event[item], delta);
265}
266
267extern unsigned long sum_zone_node_page_state(int node,
268 enum zone_stat_item item);
269extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
270extern unsigned long node_page_state(struct pglist_data *pgdat,
271 enum node_stat_item item);
272extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
273 enum node_stat_item item);
274extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
275 enum node_stat_item item);
276extern void fold_vm_numa_events(void);
277#else
278#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
279#define node_page_state(node, item) global_node_page_state(item)
280#define node_page_state_pages(node, item) global_node_page_state_pages(item)
281#define node_page_state_monotonic(node, item) global_node_page_state_monotonic(item)
282static inline void fold_vm_numa_events(void)
283{
284}
285#endif /* CONFIG_NUMA */
286
287#ifdef CONFIG_SMP
288void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
289void __inc_zone_page_state(struct page *, enum zone_stat_item);
290void __dec_zone_page_state(struct page *, enum zone_stat_item);
291
292void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
293void __inc_node_page_state(struct page *, enum node_stat_item);
294void __dec_node_page_state(struct page *, enum node_stat_item);
295
296void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
297void inc_zone_page_state(struct page *, enum zone_stat_item);
298void dec_zone_page_state(struct page *, enum zone_stat_item);
299
300void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
301void inc_node_page_state(struct page *, enum node_stat_item);
302void dec_node_page_state(struct page *, enum node_stat_item);
303
304extern void __inc_zone_state(struct zone *, enum zone_stat_item);
305extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
306extern void __dec_zone_state(struct zone *, enum zone_stat_item);
307extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
308
309void quiet_vmstat(void);
310void cpu_vm_stats_fold(int cpu);
311void refresh_zone_stat_thresholds(void);
312
313void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *);
314
315int calculate_pressure_threshold(struct zone *zone);
316int calculate_normal_threshold(struct zone *zone);
317void set_pgdat_percpu_threshold(pg_data_t *pgdat,
318 int (*calculate_pressure)(struct zone *));
319void vmstat_flush_workqueue(void);
320#else /* CONFIG_SMP */
321
322/*
323 * We do not maintain differentials in a single processor configuration.
324 * The functions directly modify the zone and global counters.
325 */
326static inline void __mod_zone_page_state(struct zone *zone,
327 enum zone_stat_item item, long delta)
328{
329 zone_page_state_add(delta, zone, item);
330}
331
332static inline void __mod_node_page_state(struct pglist_data *pgdat,
333 enum node_stat_item item, int delta)
334{
335 if (vmstat_item_in_bytes(item)) {
336 /*
337 * Only cgroups use subpage accounting right now; at
338 * the global level, these items still change in
339 * multiples of whole pages. Store them as pages
340 * internally to keep the per-cpu counters compact.
341 */
342 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
343 delta >>= PAGE_SHIFT;
344 }
345
346 node_page_state_add(delta, pgdat, item);
347}
348
349static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
350{
351 atomic_long_inc(&zone->vm_stat[item]);
352 atomic_long_inc(&vm_zone_stat[item]);
353}
354
355static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
356{
357 atomic_long_inc(&pgdat->vm_stat[item]);
358 atomic_long_inc(&vm_node_stat[item]);
359}
360
361static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
362{
363 atomic_long_dec(&zone->vm_stat[item]);
364 atomic_long_dec(&vm_zone_stat[item]);
365}
366
367static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
368{
369 atomic_long_dec(&pgdat->vm_stat[item]);
370 atomic_long_dec(&vm_node_stat[item]);
371}
372
373static inline void __inc_zone_page_state(struct page *page,
374 enum zone_stat_item item)
375{
376 __inc_zone_state(page_zone(page), item);
377}
378
379static inline void __inc_node_page_state(struct page *page,
380 enum node_stat_item item)
381{
382 __inc_node_state(page_pgdat(page), item);
383}
384
385
386static inline void __dec_zone_page_state(struct page *page,
387 enum zone_stat_item item)
388{
389 __dec_zone_state(page_zone(page), item);
390}
391
392static inline void __dec_node_page_state(struct page *page,
393 enum node_stat_item item)
394{
395 __dec_node_state(page_pgdat(page), item);
396}
397
398
399/*
400 * We only use atomic operations to update counters. So there is no need to
401 * disable interrupts.
402 */
403#define inc_zone_page_state __inc_zone_page_state
404#define dec_zone_page_state __dec_zone_page_state
405#define mod_zone_page_state __mod_zone_page_state
406
407#define inc_node_page_state __inc_node_page_state
408#define dec_node_page_state __dec_node_page_state
409#define mod_node_page_state __mod_node_page_state
410
411#define set_pgdat_percpu_threshold(pgdat, callback) { }
412
413static inline void refresh_zone_stat_thresholds(void) { }
414static inline void cpu_vm_stats_fold(int cpu) { }
415static inline void quiet_vmstat(void) { }
416static inline void vmstat_flush_workqueue(void) { }
417
418static inline void drain_zonestat(struct zone *zone,
419 struct per_cpu_zonestat *pzstats) { }
420#endif /* CONFIG_SMP */
421
422static inline void __zone_stat_mod_folio(struct folio *folio,
423 enum zone_stat_item item, long nr)
424{
425 __mod_zone_page_state(folio_zone(folio), item, nr);
426}
427
428static inline void __zone_stat_add_folio(struct folio *folio,
429 enum zone_stat_item item)
430{
431 __mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
432}
433
434static inline void __zone_stat_sub_folio(struct folio *folio,
435 enum zone_stat_item item)
436{
437 __mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
438}
439
440static inline void zone_stat_mod_folio(struct folio *folio,
441 enum zone_stat_item item, long nr)
442{
443 mod_zone_page_state(folio_zone(folio), item, nr);
444}
445
446static inline void zone_stat_add_folio(struct folio *folio,
447 enum zone_stat_item item)
448{
449 mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
450}
451
452static inline void zone_stat_sub_folio(struct folio *folio,
453 enum zone_stat_item item)
454{
455 mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
456}
457
458static inline void __node_stat_mod_folio(struct folio *folio,
459 enum node_stat_item item, long nr)
460{
461 __mod_node_page_state(folio_pgdat(folio), item, nr);
462}
463
464static inline void __node_stat_add_folio(struct folio *folio,
465 enum node_stat_item item)
466{
467 __mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
468}
469
470static inline void __node_stat_sub_folio(struct folio *folio,
471 enum node_stat_item item)
472{
473 __mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
474}
475
476static inline void node_stat_mod_folio(struct folio *folio,
477 enum node_stat_item item, long nr)
478{
479 mod_node_page_state(folio_pgdat(folio), item, nr);
480}
481
482static inline void node_stat_add_folio(struct folio *folio,
483 enum node_stat_item item)
484{
485 mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
486}
487
488static inline void node_stat_sub_folio(struct folio *folio,
489 enum node_stat_item item)
490{
491 mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
492}
493
494extern const char * const vmstat_text[];
495
496static inline const char *zone_stat_name(enum zone_stat_item item)
497{
498 return vmstat_text[item];
499}
500
501#ifdef CONFIG_NUMA
502static inline const char *numa_stat_name(enum numa_stat_item item)
503{
504 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
505 item];
506}
507#endif /* CONFIG_NUMA */
508
509static inline const char *node_stat_name(enum node_stat_item item)
510{
511 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
512 NR_VM_NUMA_EVENT_ITEMS +
513 item];
514}
515
516static inline const char *lru_list_name(enum lru_list lru)
517{
518 return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
519}
520
521#if defined(CONFIG_VM_EVENT_COUNTERS)
522static inline const char *vm_event_name(enum vm_event_item item)
523{
524 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
525 NR_VM_NUMA_EVENT_ITEMS +
526 NR_VM_NODE_STAT_ITEMS +
527 NR_VM_STAT_ITEMS +
528 item];
529}
530#endif /* CONFIG_VM_EVENT_COUNTERS */
531
532#ifdef CONFIG_MEMCG
533
534void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
535 int val);
536
537void lruvec_stat_mod_folio(struct folio *folio,
538 enum node_stat_item idx, int val);
539
540static inline void mod_lruvec_page_state(struct page *page,
541 enum node_stat_item idx, int val)
542{
543 lruvec_stat_mod_folio(page_folio(page), idx, val);
544}
545
546#else
547
548static inline void mod_lruvec_state(struct lruvec *lruvec,
549 enum node_stat_item idx, int val)
550{
551 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
552}
553
554static inline void lruvec_stat_mod_folio(struct folio *folio,
555 enum node_stat_item idx, int val)
556{
557 mod_node_page_state(folio_pgdat(folio), idx, val);
558}
559
560static inline void mod_lruvec_page_state(struct page *page,
561 enum node_stat_item idx, int val)
562{
563 mod_node_page_state(page_pgdat(page), idx, val);
564}
565
566#endif /* CONFIG_MEMCG */
567
568static inline void lruvec_stat_add_folio(struct folio *folio,
569 enum node_stat_item idx)
570{
571 lruvec_stat_mod_folio(folio, idx, folio_nr_pages(folio));
572}
573
574static inline void lruvec_stat_sub_folio(struct folio *folio,
575 enum node_stat_item idx)
576{
577 lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio));
578}
579
580void memmap_boot_pages_add(long delta);
581void memmap_pages_add(long delta);
582#endif /* _LINUX_VMSTAT_H */