projects/linux·blob master · refs

2453 lines · 64.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/vmstat.c
4 *
5 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 *
8 * zoned VM statistics
9 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <[email protected]>
11 * Copyright (C) 2008-2014 Christoph Lameter
12 */
13#include <linux/fs.h>
14#include <linux/mm.h>
15#include <linux/err.h>
16#include <linux/module.h>
17#include <linux/slab.h>
18#include <linux/cpu.h>
19#include <linux/cpumask.h>
20#include <linux/vmstat.h>
21#include <linux/proc_fs.h>
22#include <linux/seq_file.h>
23#include <linux/debugfs.h>
24#include <linux/sched.h>
25#include <linux/math64.h>
26#include <linux/writeback.h>
27#include <linux/compaction.h>
28#include <linux/mm_inline.h>
29#include <linux/page_owner.h>
30#include <linux/sched/isolation.h>
31
32#include "internal.h"
33#include "page_alloc.h"
34
35#ifdef CONFIG_PROC_FS
36#ifdef CONFIG_NUMA
37#define ENABLE_NUMA_STAT 1
38static int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
39
40/* zero numa counters within a zone */
41static void zero_zone_numa_counters(struct zone *zone)
42{
43 int item, cpu;
44
45 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) {
46 atomic_long_set(&zone->vm_numa_event[item], 0);
47 for_each_online_cpu(cpu) {
48 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]
49 = 0;
50 }
51 }
52}
53
54/* zero numa counters of all the populated zones */
55static void zero_zones_numa_counters(void)
56{
57 struct zone *zone;
58
59 for_each_populated_zone(zone)
60 zero_zone_numa_counters(zone);
61}
62
63/* zero global numa counters */
64static void zero_global_numa_counters(void)
65{
66 int item;
67
68 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++)
69 atomic_long_set(&vm_numa_event[item], 0);
70}
71
72static void invalid_numa_statistics(void)
73{
74 zero_zones_numa_counters();
75 zero_global_numa_counters();
76}
77
78static DEFINE_MUTEX(vm_numa_stat_lock);
79
80static int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write,
81 void *buffer, size_t *length, loff_t *ppos)
82{
83 int ret, oldval;
84
85 mutex_lock(&vm_numa_stat_lock);
86 if (write)
87 oldval = sysctl_vm_numa_stat;
88 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
89 if (ret || !write)
90 goto out;
91
92 if (oldval == sysctl_vm_numa_stat)
93 goto out;
94 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
95 static_branch_enable(&vm_numa_stat_key);
96 pr_info("enable numa statistics\n");
97 } else {
98 static_branch_disable(&vm_numa_stat_key);
99 invalid_numa_statistics();
100 pr_info("disable numa statistics, and clear numa counters\n");
101 }
102
103out:
104 mutex_unlock(&vm_numa_stat_lock);
105 return ret;
106}
107#endif
108#endif /* CONFIG_PROC_FS */
109
110#ifdef CONFIG_VM_EVENT_COUNTERS
111DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
112EXPORT_PER_CPU_SYMBOL(vm_event_states);
113
114static void sum_vm_events(unsigned long *ret)
115{
116 int cpu;
117 int i;
118
119 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
120
121 for_each_online_cpu(cpu) {
122 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
123
124 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
125 ret[i] += this->event[i];
126 }
127}
128
129/*
130 * Accumulate the vm event counters across all CPUs.
131 * The result is unavoidably approximate - it can change
132 * during and after execution of this function.
133*/
134void all_vm_events(unsigned long *ret)
135{
136 cpus_read_lock();
137 sum_vm_events(ret);
138 cpus_read_unlock();
139}
140EXPORT_SYMBOL_GPL(all_vm_events);
141
142/*
143 * Fold the foreign cpu events into our own.
144 *
145 * This is adding to the events on one processor
146 * but keeps the global counts constant.
147 */
148void vm_events_fold_cpu(int cpu)
149{
150 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
151 int i;
152
153 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
154 count_vm_events(i, fold_state->event[i]);
155 fold_state->event[i] = 0;
156 }
157}
158
159#endif /* CONFIG_VM_EVENT_COUNTERS */
160
161/*
162 * Manage combined zone based / global counters
163 *
164 * vm_stat contains the global counters
165 */
166atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
167atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
168atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp;
169EXPORT_SYMBOL(vm_zone_stat);
170EXPORT_SYMBOL(vm_node_stat);
171
172#ifdef CONFIG_NUMA
173static void fold_vm_zone_numa_events(struct zone *zone)
174{
175 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, };
176 int cpu;
177 enum numa_stat_item item;
178
179 for_each_online_cpu(cpu) {
180 struct per_cpu_zonestat *pzstats;
181
182 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
183 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++)
184 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0);
185 }
186
187 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++)
188 zone_numa_event_add(zone_numa_events[item], zone, item);
189}
190
191void fold_vm_numa_events(void)
192{
193 struct zone *zone;
194
195 for_each_populated_zone(zone)
196 fold_vm_zone_numa_events(zone);
197}
198#endif
199
200#ifdef CONFIG_SMP
201
202int calculate_pressure_threshold(struct zone *zone)
203{
204 int threshold;
205 int watermark_distance;
206
207 /*
208 * As vmstats are not up to date, there is drift between the estimated
209 * and real values. For high thresholds and a high number of CPUs, it
210 * is possible for the min watermark to be breached while the estimated
211 * value looks fine. The pressure threshold is a reduced value such
212 * that even the maximum amount of drift will not accidentally breach
213 * the min watermark
214 */
215 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
216 threshold = max(1, (int)(watermark_distance / num_online_cpus()));
217
218 /*
219 * Maximum threshold is 125
220 */
221 threshold = min(125, threshold);
222
223 return threshold;
224}
225
226int calculate_normal_threshold(struct zone *zone)
227{
228 int threshold;
229 int mem; /* memory in 128 MB units */
230
231 /*
232 * The threshold scales with the number of processors and the amount
233 * of memory per zone. More memory means that we can defer updates for
234 * longer, more processors could lead to more contention.
235 * fls() is used to have a cheap way of logarithmic scaling.
236 *
237 * Some sample thresholds:
238 *
239 * Threshold Processors (fls) Zonesize fls(mem)+1
240 * ------------------------------------------------------------------
241 * 8 1 1 0.9-1 GB 4
242 * 16 2 2 0.9-1 GB 4
243 * 20 2 2 1-2 GB 5
244 * 24 2 2 2-4 GB 6
245 * 28 2 2 4-8 GB 7
246 * 32 2 2 8-16 GB 8
247 * 4 2 2 <128M 1
248 * 30 4 3 2-4 GB 5
249 * 48 4 3 8-16 GB 8
250 * 32 8 4 1-2 GB 4
251 * 32 8 4 0.9-1GB 4
252 * 10 16 5 <128M 1
253 * 40 16 5 900M 4
254 * 70 64 7 2-4 GB 5
255 * 84 64 7 4-8 GB 6
256 * 108 512 9 4-8 GB 6
257 * 125 1024 10 8-16 GB 8
258 * 125 1024 10 16-32 GB 9
259 */
260
261 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);
262
263 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
264
265 /*
266 * Maximum threshold is 125
267 */
268 threshold = min(125, threshold);
269
270 return threshold;
271}
272
273/*
274 * Refresh the thresholds for each zone.
275 */
276void refresh_zone_stat_thresholds(void)
277{
278 struct pglist_data *pgdat;
279 struct zone *zone;
280 int cpu;
281 int threshold;
282
283 /* Zero current pgdat thresholds */
284 for_each_online_pgdat(pgdat) {
285 for_each_online_cpu(cpu) {
286 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
287 }
288 }
289
290 for_each_populated_zone(zone) {
291 struct pglist_data *pgdat = zone->zone_pgdat;
292 unsigned long max_drift, tolerate_drift;
293
294 threshold = calculate_normal_threshold(zone);
295
296 for_each_online_cpu(cpu) {
297 int pgdat_threshold;
298
299 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold
300 = threshold;
301
302 /* Base nodestat threshold on the largest populated zone. */
303 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
304 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
305 = max(threshold, pgdat_threshold);
306 }
307
308 /*
309 * Only set percpu_drift_mark if there is a danger that
310 * NR_FREE_PAGES reports the low watermark is ok when in fact
311 * the min watermark could be breached by an allocation
312 */
313 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
314 max_drift = num_online_cpus() * threshold;
315 if (max_drift > tolerate_drift)
316 zone->percpu_drift_mark = high_wmark_pages(zone) +
317 max_drift;
318 }
319}
320
321void set_pgdat_percpu_threshold(pg_data_t *pgdat,
322 int (*calculate_pressure)(struct zone *))
323{
324 struct zone *zone;
325 int cpu;
326 int threshold;
327 int i;
328
329 for (i = 0; i < pgdat->nr_zones; i++) {
330 zone = &pgdat->node_zones[i];
331 if (!zone->percpu_drift_mark)
332 continue;
333
334 threshold = (*calculate_pressure)(zone);
335 for_each_online_cpu(cpu)
336 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold
337 = threshold;
338 }
339}
340
341/*
342 * For use when we know that interrupts are disabled,
343 * or when we know that preemption is disabled and that
344 * particular counter cannot be updated from interrupt context.
345 */
346void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
347 long delta)
348{
349 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
350 s8 __percpu *p = pcp->vm_stat_diff + item;
351 long x;
352 long t;
353
354 /*
355 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels,
356 * atomicity is provided by IRQs being disabled -- either explicitly
357 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables
358 * CPU migrations and preemption potentially corrupts a counter so
359 * disable preemption.
360 */
361 preempt_disable_nested();
362
363 x = delta + __this_cpu_read(*p);
364
365 t = __this_cpu_read(pcp->stat_threshold);
366
367 if (unlikely(abs(x) > t)) {
368 zone_page_state_add(x, zone, item);
369 x = 0;
370 }
371 __this_cpu_write(*p, x);
372
373 preempt_enable_nested();
374}
375EXPORT_SYMBOL(__mod_zone_page_state);
376
377void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
378 long delta)
379{
380 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
381 s8 __percpu *p = pcp->vm_node_stat_diff + item;
382 long x;
383 long t;
384
385 if (vmstat_item_in_bytes(item)) {
386 /*
387 * Only cgroups use subpage accounting right now; at
388 * the global level, these items still change in
389 * multiples of whole pages. Store them as pages
390 * internally to keep the per-cpu counters compact.
391 */
392 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
393 delta >>= PAGE_SHIFT;
394 }
395
396 /* See __mod_zone_page_state() */
397 preempt_disable_nested();
398
399 x = delta + __this_cpu_read(*p);
400
401 t = __this_cpu_read(pcp->stat_threshold);
402
403 if (unlikely(abs(x) > t)) {
404 node_page_state_add(x, pgdat, item);
405 x = 0;
406 }
407 __this_cpu_write(*p, x);
408
409 preempt_enable_nested();
410}
411EXPORT_SYMBOL(__mod_node_page_state);
412
413/*
414 * Optimized increment and decrement functions.
415 *
416 * These are only for a single page and therefore can take a struct page *
417 * argument instead of struct zone *. This allows the inclusion of the code
418 * generated for page_zone(page) into the optimized functions.
419 *
420 * No overflow check is necessary and therefore the differential can be
421 * incremented or decremented in place which may allow the compilers to
422 * generate better code.
423 * The increment or decrement is known and therefore one boundary check can
424 * be omitted.
425 *
426 * NOTE: These functions are very performance sensitive. Change only
427 * with care.
428 *
429 * Some processors have inc/dec instructions that are atomic vs an interrupt.
430 * However, the code must first determine the differential location in a zone
431 * based on the processor number and then inc/dec the counter. There is no
432 * guarantee without disabling preemption that the processor will not change
433 * in between and therefore the atomicity vs. interrupt cannot be exploited
434 * in a useful way here.
435 */
436void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
437{
438 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
439 s8 __percpu *p = pcp->vm_stat_diff + item;
440 s8 v, t;
441
442 /* See __mod_zone_page_state() */
443 preempt_disable_nested();
444
445 v = __this_cpu_inc_return(*p);
446 t = __this_cpu_read(pcp->stat_threshold);
447 if (unlikely(v > t)) {
448 s8 overstep = t >> 1;
449
450 zone_page_state_add(v + overstep, zone, item);
451 __this_cpu_write(*p, -overstep);
452 }
453
454 preempt_enable_nested();
455}
456
457void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
458{
459 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
460 s8 __percpu *p = pcp->vm_node_stat_diff + item;
461 s8 v, t;
462
463 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
464
465 /* See __mod_zone_page_state() */
466 preempt_disable_nested();
467
468 v = __this_cpu_inc_return(*p);
469 t = __this_cpu_read(pcp->stat_threshold);
470 if (unlikely(v > t)) {
471 s8 overstep = t >> 1;
472
473 node_page_state_add(v + overstep, pgdat, item);
474 __this_cpu_write(*p, -overstep);
475 }
476
477 preempt_enable_nested();
478}
479
480void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
481{
482 __inc_zone_state(page_zone(page), item);
483}
484EXPORT_SYMBOL(__inc_zone_page_state);
485
486void __inc_node_page_state(struct page *page, enum node_stat_item item)
487{
488 __inc_node_state(page_pgdat(page), item);
489}
490EXPORT_SYMBOL(__inc_node_page_state);
491
492void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
493{
494 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
495 s8 __percpu *p = pcp->vm_stat_diff + item;
496 s8 v, t;
497
498 /* See __mod_zone_page_state() */
499 preempt_disable_nested();
500
501 v = __this_cpu_dec_return(*p);
502 t = __this_cpu_read(pcp->stat_threshold);
503 if (unlikely(v < - t)) {
504 s8 overstep = t >> 1;
505
506 zone_page_state_add(v - overstep, zone, item);
507 __this_cpu_write(*p, overstep);
508 }
509
510 preempt_enable_nested();
511}
512
513void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
514{
515 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
516 s8 __percpu *p = pcp->vm_node_stat_diff + item;
517 s8 v, t;
518
519 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
520
521 /* See __mod_zone_page_state() */
522 preempt_disable_nested();
523
524 v = __this_cpu_dec_return(*p);
525 t = __this_cpu_read(pcp->stat_threshold);
526 if (unlikely(v < - t)) {
527 s8 overstep = t >> 1;
528
529 node_page_state_add(v - overstep, pgdat, item);
530 __this_cpu_write(*p, overstep);
531 }
532
533 preempt_enable_nested();
534}
535
536void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
537{
538 __dec_zone_state(page_zone(page), item);
539}
540EXPORT_SYMBOL(__dec_zone_page_state);
541
542void __dec_node_page_state(struct page *page, enum node_stat_item item)
543{
544 __dec_node_state(page_pgdat(page), item);
545}
546EXPORT_SYMBOL(__dec_node_page_state);
547
548#ifdef CONFIG_HAVE_CMPXCHG_LOCAL
549/*
550 * If we have cmpxchg_local support then we do not need to incur the overhead
551 * that comes with local_irq_save/restore if we use this_cpu_try_cmpxchg().
552 *
553 * mod_state() modifies the zone counter state through atomic per cpu
554 * operations.
555 *
556 * Overstep mode specifies how overstep should handled:
557 * 0 No overstepping
558 * 1 Overstepping half of threshold
559 * -1 Overstepping minus half of threshold
560*/
561static inline void mod_zone_state(struct zone *zone,
562 enum zone_stat_item item, long delta, int overstep_mode)
563{
564 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
565 s8 __percpu *p = pcp->vm_stat_diff + item;
566 long n, t, z;
567 s8 o;
568
569 o = this_cpu_read(*p);
570 do {
571 z = 0; /* overflow to zone counters */
572
573 /*
574 * The fetching of the stat_threshold is racy. We may apply
575 * a counter threshold to the wrong the cpu if we get
576 * rescheduled while executing here. However, the next
577 * counter update will apply the threshold again and
578 * therefore bring the counter under the threshold again.
579 *
580 * Most of the time the thresholds are the same anyways
581 * for all cpus in a zone.
582 */
583 t = this_cpu_read(pcp->stat_threshold);
584
585 n = delta + (long)o;
586
587 if (abs(n) > t) {
588 int os = overstep_mode * (t >> 1) ;
589
590 /* Overflow must be added to zone counters */
591 z = n + os;
592 n = -os;
593 }
594 } while (!this_cpu_try_cmpxchg(*p, &o, n));
595
596 if (z)
597 zone_page_state_add(z, zone, item);
598}
599
600void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
601 long delta)
602{
603 mod_zone_state(zone, item, delta, 0);
604}
605EXPORT_SYMBOL(mod_zone_page_state);
606
607void inc_zone_page_state(struct page *page, enum zone_stat_item item)
608{
609 mod_zone_state(page_zone(page), item, 1, 1);
610}
611EXPORT_SYMBOL(inc_zone_page_state);
612
613void dec_zone_page_state(struct page *page, enum zone_stat_item item)
614{
615 mod_zone_state(page_zone(page), item, -1, -1);
616}
617EXPORT_SYMBOL(dec_zone_page_state);
618
619static inline void mod_node_state(struct pglist_data *pgdat,
620 enum node_stat_item item, int delta, int overstep_mode)
621{
622 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
623 s8 __percpu *p = pcp->vm_node_stat_diff + item;
624 long n, t, z;
625 s8 o;
626
627 if (vmstat_item_in_bytes(item)) {
628 /*
629 * Only cgroups use subpage accounting right now; at
630 * the global level, these items still change in
631 * multiples of whole pages. Store them as pages
632 * internally to keep the per-cpu counters compact.
633 */
634 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
635 delta >>= PAGE_SHIFT;
636 }
637
638 o = this_cpu_read(*p);
639 do {
640 z = 0; /* overflow to node counters */
641
642 /*
643 * The fetching of the stat_threshold is racy. We may apply
644 * a counter threshold to the wrong the cpu if we get
645 * rescheduled while executing here. However, the next
646 * counter update will apply the threshold again and
647 * therefore bring the counter under the threshold again.
648 *
649 * Most of the time the thresholds are the same anyways
650 * for all cpus in a node.
651 */
652 t = this_cpu_read(pcp->stat_threshold);
653
654 n = delta + (long)o;
655
656 if (abs(n) > t) {
657 int os = overstep_mode * (t >> 1) ;
658
659 /* Overflow must be added to node counters */
660 z = n + os;
661 n = -os;
662 }
663 } while (!this_cpu_try_cmpxchg(*p, &o, n));
664
665 if (z)
666 node_page_state_add(z, pgdat, item);
667}
668
669void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
670 long delta)
671{
672 mod_node_state(pgdat, item, delta, 0);
673}
674EXPORT_SYMBOL(mod_node_page_state);
675
676void inc_node_page_state(struct page *page, enum node_stat_item item)
677{
678 mod_node_state(page_pgdat(page), item, 1, 1);
679}
680EXPORT_SYMBOL(inc_node_page_state);
681
682void dec_node_page_state(struct page *page, enum node_stat_item item)
683{
684 mod_node_state(page_pgdat(page), item, -1, -1);
685}
686EXPORT_SYMBOL(dec_node_page_state);
687#else
688/*
689 * Use interrupt disable to serialize counter updates
690 */
691void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
692 long delta)
693{
694 unsigned long flags;
695
696 local_irq_save(flags);
697 __mod_zone_page_state(zone, item, delta);
698 local_irq_restore(flags);
699}
700EXPORT_SYMBOL(mod_zone_page_state);
701
702void inc_zone_page_state(struct page *page, enum zone_stat_item item)
703{
704 unsigned long flags;
705 struct zone *zone;
706
707 zone = page_zone(page);
708 local_irq_save(flags);
709 __inc_zone_state(zone, item);
710 local_irq_restore(flags);
711}
712EXPORT_SYMBOL(inc_zone_page_state);
713
714void dec_zone_page_state(struct page *page, enum zone_stat_item item)
715{
716 unsigned long flags;
717
718 local_irq_save(flags);
719 __dec_zone_page_state(page, item);
720 local_irq_restore(flags);
721}
722EXPORT_SYMBOL(dec_zone_page_state);
723
724void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
725 long delta)
726{
727 unsigned long flags;
728
729 local_irq_save(flags);
730 __mod_node_page_state(pgdat, item, delta);
731 local_irq_restore(flags);
732}
733EXPORT_SYMBOL(mod_node_page_state);
734
735void inc_node_page_state(struct page *page, enum node_stat_item item)
736{
737 unsigned long flags;
738 struct pglist_data *pgdat;
739
740 pgdat = page_pgdat(page);
741 local_irq_save(flags);
742 __inc_node_state(pgdat, item);
743 local_irq_restore(flags);
744}
745EXPORT_SYMBOL(inc_node_page_state);
746
747void dec_node_page_state(struct page *page, enum node_stat_item item)
748{
749 unsigned long flags;
750
751 local_irq_save(flags);
752 __dec_node_page_state(page, item);
753 local_irq_restore(flags);
754}
755EXPORT_SYMBOL(dec_node_page_state);
756#endif
757
758/*
759 * Fold a differential into the global counters.
760 * Returns whether counters were updated.
761 */
762static int fold_diff(int *zone_diff, int *node_diff)
763{
764 int i;
765 bool changed = false;
766
767 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
768 if (zone_diff[i]) {
769 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
770 changed = true;
771 }
772 }
773
774 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
775 if (node_diff[i]) {
776 atomic_long_add(node_diff[i], &vm_node_stat[i]);
777 changed = true;
778 }
779 }
780
781 return changed;
782}
783
784/*
785 * Update the zone counters for the current cpu.
786 *
787 * Note that refresh_cpu_vm_stats strives to only access
788 * node local memory. The per cpu pagesets on remote zones are placed
789 * in the memory local to the processor using that pageset. So the
790 * loop over all zones will access a series of cachelines local to
791 * the processor.
792 *
793 * The call to zone_page_state_add updates the cachelines with the
794 * statistics in the remote zone struct as well as the global cachelines
795 * with the global counters. These could cause remote node cache line
796 * bouncing and will have to be only done when necessary.
797 *
798 * The function returns whether global counters were updated.
799 */
800static bool refresh_cpu_vm_stats(bool do_pagesets)
801{
802 struct pglist_data *pgdat;
803 struct zone *zone;
804 int i;
805 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
806 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
807 bool changed = false;
808
809 for_each_populated_zone(zone) {
810 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
811 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;
812
813 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
814 int v;
815
816 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0);
817 if (v) {
818
819 atomic_long_add(v, &zone->vm_stat[i]);
820 global_zone_diff[i] += v;
821#ifdef CONFIG_NUMA
822 /* 3 seconds idle till flush */
823 __this_cpu_write(pcp->expire, 3);
824#endif
825 }
826 }
827
828 if (do_pagesets) {
829 cond_resched();
830
831 if (decay_pcp_high(zone, this_cpu_ptr(pcp)))
832 changed = true;
833#ifdef CONFIG_NUMA
834 /*
835 * Deal with draining the remote pageset of this
836 * processor
837 *
838 * Check if there are pages remaining in this pageset
839 * if not then there is nothing to expire.
840 */
841 if (!__this_cpu_read(pcp->expire) ||
842 !__this_cpu_read(pcp->count))
843 continue;
844
845 /*
846 * We never drain zones local to this processor.
847 */
848 if (zone_to_nid(zone) == numa_node_id()) {
849 __this_cpu_write(pcp->expire, 0);
850 continue;
851 }
852
853 if (__this_cpu_dec_return(pcp->expire)) {
854 changed = true;
855 continue;
856 }
857
858 if (__this_cpu_read(pcp->count)) {
859 drain_zone_pages(zone, this_cpu_ptr(pcp));
860 changed = true;
861 }
862#endif
863 }
864 }
865
866 for_each_online_pgdat(pgdat) {
867 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
868
869 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
870 int v;
871
872 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
873 if (v) {
874 atomic_long_add(v, &pgdat->vm_stat[i]);
875 global_node_diff[i] += v;
876 }
877 }
878 }
879
880 if (fold_diff(global_zone_diff, global_node_diff))
881 changed = true;
882 return changed;
883}
884
885/*
886 * Fold the data for an offline cpu into the global array.
887 * There cannot be any access by the offline cpu and therefore
888 * synchronization is simplified.
889 */
890void cpu_vm_stats_fold(int cpu)
891{
892 struct pglist_data *pgdat;
893 struct zone *zone;
894 int i;
895 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
896 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
897
898 for_each_populated_zone(zone) {
899 struct per_cpu_zonestat *pzstats;
900
901 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
902
903 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
904 if (pzstats->vm_stat_diff[i]) {
905 int v;
906
907 v = pzstats->vm_stat_diff[i];
908 pzstats->vm_stat_diff[i] = 0;
909 atomic_long_add(v, &zone->vm_stat[i]);
910 global_zone_diff[i] += v;
911 }
912 }
913#ifdef CONFIG_NUMA
914 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) {
915 if (pzstats->vm_numa_event[i]) {
916 unsigned long v;
917
918 v = pzstats->vm_numa_event[i];
919 pzstats->vm_numa_event[i] = 0;
920 zone_numa_event_add(v, zone, i);
921 }
922 }
923#endif
924 }
925
926 for_each_online_pgdat(pgdat) {
927 struct per_cpu_nodestat *p;
928
929 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
930
931 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
932 if (p->vm_node_stat_diff[i]) {
933 int v;
934
935 v = p->vm_node_stat_diff[i];
936 p->vm_node_stat_diff[i] = 0;
937 atomic_long_add(v, &pgdat->vm_stat[i]);
938 global_node_diff[i] += v;
939 }
940 }
941
942 fold_diff(global_zone_diff, global_node_diff);
943}
944
945/*
946 * this is only called if !populated_zone(zone), which implies no other users of
947 * pset->vm_stat_diff[] exist.
948 */
949void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)
950{
951 unsigned long v;
952 int i;
953
954 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
955 if (pzstats->vm_stat_diff[i]) {
956 v = pzstats->vm_stat_diff[i];
957 pzstats->vm_stat_diff[i] = 0;
958 zone_page_state_add(v, zone, i);
959 }
960 }
961
962#ifdef CONFIG_NUMA
963 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) {
964 if (pzstats->vm_numa_event[i]) {
965 v = pzstats->vm_numa_event[i];
966 pzstats->vm_numa_event[i] = 0;
967 zone_numa_event_add(v, zone, i);
968 }
969 }
970#endif
971}
972#endif
973
974#ifdef CONFIG_NUMA
975/*
976 * Determine the per node value of a stat item. This function
977 * is called frequently in a NUMA machine, so try to be as
978 * frugal as possible.
979 */
980unsigned long sum_zone_node_page_state(int node,
981 enum zone_stat_item item)
982{
983 struct zone *zones = NODE_DATA(node)->node_zones;
984 int i;
985 unsigned long count = 0;
986
987 for (i = 0; i < MAX_NR_ZONES; i++)
988 count += zone_page_state(zones + i, item);
989
990 return count;
991}
992
993/* Determine the per node value of a numa stat item. */
994unsigned long sum_zone_numa_event_state(int node,
995 enum numa_stat_item item)
996{
997 struct zone *zones = NODE_DATA(node)->node_zones;
998 unsigned long count = 0;
999 int i;
1000
1001 for (i = 0; i < MAX_NR_ZONES; i++)
1002 count += zone_numa_event_state(zones + i, item);
1003
1004 return count;
1005}
1006
1007/*
1008 * Determine the per node value of a stat item.
1009 */
1010unsigned long node_page_state_pages(struct pglist_data *pgdat,
1011 enum node_stat_item item)
1012{
1013 long x = atomic_long_read(&pgdat->vm_stat[item]);
1014#ifdef CONFIG_SMP
1015 if (x < 0)
1016 x = 0;
1017#endif
1018 return x;
1019}
1020
1021unsigned long node_page_state(struct pglist_data *pgdat,
1022 enum node_stat_item item)
1023{
1024 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
1025
1026 return node_page_state_pages(pgdat, item);
1027}
1028
1029/*
1030 * Non-clamping variant of node_page_state() intended for callers that
1031 * snapshot a monotonically-incremented counter and subtract two samples.
1032 * See global_node_page_state_monotonic() for the rationale.
1033 */
1034unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
1035 enum node_stat_item item)
1036{
1037 return (unsigned long)atomic_long_read(&pgdat->vm_stat[item]);
1038}
1039#endif
1040
1041/*
1042 * Count number of pages "struct page" and "struct page_ext" consume.
1043 * nr_memmap_boot_pages: # of pages allocated by boot allocator
1044 * nr_memmap_pages: # of pages that were allocated by buddy allocator
1045 */
1046static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0);
1047static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0);
1048
1049void memmap_boot_pages_add(long delta)
1050{
1051 atomic_long_add(delta, &nr_memmap_boot_pages);
1052}
1053
1054void memmap_pages_add(long delta)
1055{
1056 atomic_long_add(delta, &nr_memmap_pages);
1057}
1058
1059#ifdef CONFIG_COMPACTION
1060
1061struct contig_page_info {
1062 unsigned long free_pages;
1063 unsigned long free_blocks_total;
1064 unsigned long free_blocks_suitable;
1065};
1066
1067/*
1068 * Calculate the number of free pages in a zone, how many contiguous
1069 * pages are free and how many are large enough to satisfy an allocation of
1070 * the target size. Note that this function makes no attempt to estimate
1071 * how many suitable free blocks there *might* be if MOVABLE pages were
1072 * migrated. Calculating that is possible, but expensive and can be
1073 * figured out from userspace
1074 */
1075static void fill_contig_page_info(struct zone *zone,
1076 unsigned int suitable_order,
1077 struct contig_page_info *info)
1078{
1079 unsigned int order;
1080
1081 info->free_pages = 0;
1082 info->free_blocks_total = 0;
1083 info->free_blocks_suitable = 0;
1084
1085 for (order = 0; order < NR_PAGE_ORDERS; order++) {
1086 unsigned long blocks;
1087
1088 /*
1089 * Count number of free blocks.
1090 *
1091 * Access to nr_free is lockless as nr_free is used only for
1092 * diagnostic purposes. Use data_race to avoid KCSAN warning.
1093 */
1094 blocks = data_race(zone->free_area[order].nr_free);
1095 info->free_blocks_total += blocks;
1096
1097 /* Count free base pages */
1098 info->free_pages += blocks << order;
1099
1100 /* Count the suitable free blocks */
1101 if (order >= suitable_order)
1102 info->free_blocks_suitable += blocks <<
1103 (order - suitable_order);
1104 }
1105}
1106
1107/*
1108 * A fragmentation index only makes sense if an allocation of a requested
1109 * size would fail. If that is true, the fragmentation index indicates
1110 * whether external fragmentation or a lack of memory was the problem.
1111 * The value can be used to determine if page reclaim or compaction
1112 * should be used
1113 */
1114static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
1115{
1116 unsigned long requested = 1UL << order;
1117
1118 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER))
1119 return 0;
1120
1121 if (!info->free_blocks_total)
1122 return 0;
1123
1124 /* Fragmentation index only makes sense when a request would fail */
1125 if (info->free_blocks_suitable)
1126 return -1000;
1127
1128 /*
1129 * Index is between 0 and 1 so return within 3 decimal places
1130 *
1131 * 0 => allocation would fail due to lack of memory
1132 * 1 => allocation would fail due to fragmentation
1133 */
1134 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
1135}
1136
1137/*
1138 * Calculates external fragmentation within a zone wrt the given order.
1139 * It is defined as the percentage of pages found in blocks of size
1140 * less than 1 << order. It returns values in range [0, 100].
1141 */
1142unsigned int extfrag_for_order(struct zone *zone, unsigned int order)
1143{
1144 struct contig_page_info info;
1145
1146 fill_contig_page_info(zone, order, &info);
1147 if (info.free_pages == 0)
1148 return 0;
1149
1150 return div_u64((info.free_pages -
1151 (info.free_blocks_suitable << order)) * 100,
1152 info.free_pages);
1153}
1154
1155/* Same as __fragmentation index but allocs contig_page_info on stack */
1156int fragmentation_index(struct zone *zone, unsigned int order)
1157{
1158 struct contig_page_info info;
1159
1160 fill_contig_page_info(zone, order, &info);
1161 return __fragmentation_index(order, &info);
1162}
1163#endif
1164
1165#if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \
1166 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG)
1167#ifdef CONFIG_ZONE_DMA
1168#define TEXT_FOR_DMA(xx, yy) [xx##_DMA] = yy "_dma",
1169#else
1170#define TEXT_FOR_DMA(xx, yy)
1171#endif
1172
1173#ifdef CONFIG_ZONE_DMA32
1174#define TEXT_FOR_DMA32(xx, yy) [xx##_DMA32] = yy "_dma32",
1175#else
1176#define TEXT_FOR_DMA32(xx, yy)
1177#endif
1178
1179#ifdef CONFIG_HIGHMEM
1180#define TEXT_FOR_HIGHMEM(xx, yy) [xx##_HIGH] = yy "_high",
1181#else
1182#define TEXT_FOR_HIGHMEM(xx, yy)
1183#endif
1184
1185#ifdef CONFIG_ZONE_DEVICE
1186#define TEXT_FOR_DEVICE(xx, yy) [xx##_DEVICE] = yy "_device",
1187#else
1188#define TEXT_FOR_DEVICE(xx, yy)
1189#endif
1190
1191#define TEXTS_FOR_ZONES(xx, yy) \
1192 TEXT_FOR_DMA(xx, yy) \
1193 TEXT_FOR_DMA32(xx, yy) \
1194 [xx##_NORMAL] = yy "_normal", \
1195 TEXT_FOR_HIGHMEM(xx, yy) \
1196 [xx##_MOVABLE] = yy "_movable", \
1197 TEXT_FOR_DEVICE(xx, yy)
1198
1199const char * const vmstat_text[] = {
1200 /* enum zone_stat_item counters */
1201#define I(x) (x)
1202 [I(NR_FREE_PAGES)] = "nr_free_pages",
1203 [I(NR_FREE_PAGES_BLOCKS)] = "nr_free_pages_blocks",
1204 [I(NR_ZONE_INACTIVE_ANON)] = "nr_zone_inactive_anon",
1205 [I(NR_ZONE_ACTIVE_ANON)] = "nr_zone_active_anon",
1206 [I(NR_ZONE_INACTIVE_FILE)] = "nr_zone_inactive_file",
1207 [I(NR_ZONE_ACTIVE_FILE)] = "nr_zone_active_file",
1208 [I(NR_ZONE_UNEVICTABLE)] = "nr_zone_unevictable",
1209 [I(NR_ZONE_WRITE_PENDING)] = "nr_zone_write_pending",
1210 [I(NR_MLOCK)] = "nr_mlock",
1211#if IS_ENABLED(CONFIG_ZSMALLOC)
1212 [I(NR_ZSPAGES)] = "nr_zspages",
1213#endif
1214 [I(NR_FREE_CMA_PAGES)] = "nr_free_cma",
1215#ifdef CONFIG_UNACCEPTED_MEMORY
1216 [I(NR_UNACCEPTED)] = "nr_unaccepted",
1217#endif
1218#undef I
1219
1220 /* enum numa_stat_item counters */
1221#define I(x) (NR_VM_ZONE_STAT_ITEMS + x)
1222#ifdef CONFIG_NUMA
1223 [I(NUMA_HIT)] = "numa_hit",
1224 [I(NUMA_MISS)] = "numa_miss",
1225 [I(NUMA_FOREIGN)] = "numa_foreign",
1226 [I(NUMA_INTERLEAVE_HIT)] = "numa_interleave",
1227 [I(NUMA_LOCAL)] = "numa_local",
1228 [I(NUMA_OTHER)] = "numa_other",
1229#endif
1230#undef I
1231
1232 /* enum node_stat_item counters */
1233#define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + x)
1234 [I(NR_INACTIVE_ANON)] = "nr_inactive_anon",
1235 [I(NR_ACTIVE_ANON)] = "nr_active_anon",
1236 [I(NR_INACTIVE_FILE)] = "nr_inactive_file",
1237 [I(NR_ACTIVE_FILE)] = "nr_active_file",
1238 [I(NR_UNEVICTABLE)] = "nr_unevictable",
1239 [I(NR_SLAB_RECLAIMABLE_B)] = "nr_slab_reclaimable",
1240 [I(NR_SLAB_UNRECLAIMABLE_B)] = "nr_slab_unreclaimable",
1241 [I(NR_ISOLATED_ANON)] = "nr_isolated_anon",
1242 [I(NR_ISOLATED_FILE)] = "nr_isolated_file",
1243 [I(WORKINGSET_NODES)] = "workingset_nodes",
1244 [I(WORKINGSET_REFAULT_ANON)] = "workingset_refault_anon",
1245 [I(WORKINGSET_REFAULT_FILE)] = "workingset_refault_file",
1246 [I(WORKINGSET_ACTIVATE_ANON)] = "workingset_activate_anon",
1247 [I(WORKINGSET_ACTIVATE_FILE)] = "workingset_activate_file",
1248 [I(WORKINGSET_RESTORE_ANON)] = "workingset_restore_anon",
1249 [I(WORKINGSET_RESTORE_FILE)] = "workingset_restore_file",
1250 [I(WORKINGSET_NODERECLAIM)] = "workingset_nodereclaim",
1251 [I(NR_ANON_MAPPED)] = "nr_anon_pages",
1252 [I(NR_FILE_MAPPED)] = "nr_mapped",
1253 [I(NR_FILE_PAGES)] = "nr_file_pages",
1254 [I(NR_FILE_DIRTY)] = "nr_dirty",
1255 [I(NR_WRITEBACK)] = "nr_writeback",
1256 [I(NR_SHMEM)] = "nr_shmem",
1257 [I(NR_SHMEM_THPS)] = "nr_shmem_hugepages",
1258 [I(NR_SHMEM_PMDMAPPED)] = "nr_shmem_pmdmapped",
1259 [I(NR_FILE_THPS)] = "nr_file_hugepages",
1260 [I(NR_FILE_PMDMAPPED)] = "nr_file_pmdmapped",
1261 [I(NR_ANON_THPS)] = "nr_anon_transparent_hugepages",
1262 [I(NR_VMSCAN_WRITE)] = "nr_vmscan_write",
1263 [I(NR_VMSCAN_IMMEDIATE)] = "nr_vmscan_immediate_reclaim",
1264 [I(NR_DIRTIED)] = "nr_dirtied",
1265 [I(NR_WRITTEN)] = "nr_written",
1266 [I(NR_THROTTLED_WRITTEN)] = "nr_throttled_written",
1267 [I(NR_KERNEL_MISC_RECLAIMABLE)] = "nr_kernel_misc_reclaimable",
1268 [I(NR_FOLL_PIN_ACQUIRED)] = "nr_foll_pin_acquired",
1269 [I(NR_FOLL_PIN_RELEASED)] = "nr_foll_pin_released",
1270 [I(NR_VMALLOC)] = "nr_vmalloc",
1271 [I(NR_KERNEL_STACK_KB)] = "nr_kernel_stack",
1272#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
1273 [I(NR_KERNEL_SCS_KB)] = "nr_shadow_call_stack",
1274#endif
1275 [I(NR_PAGETABLE)] = "nr_page_table_pages",
1276 [I(NR_SECONDARY_PAGETABLE)] = "nr_sec_page_table_pages",
1277#ifdef CONFIG_IOMMU_SUPPORT
1278 [I(NR_IOMMU_PAGES)] = "nr_iommu_pages",
1279#endif
1280#ifdef CONFIG_SWAP
1281 [I(NR_SWAPCACHE)] = "nr_swapcached",
1282#endif
1283#ifdef CONFIG_NUMA_BALANCING
1284 [I(PGPROMOTE_SUCCESS)] = "pgpromote_success",
1285 [I(PGPROMOTE_CANDIDATE)] = "pgpromote_candidate",
1286 [I(PGPROMOTE_CANDIDATE_NRL)] = "pgpromote_candidate_nrl",
1287#endif
1288 [I(PGDEMOTE_KSWAPD)] = "pgdemote_kswapd",
1289 [I(PGDEMOTE_DIRECT)] = "pgdemote_direct",
1290 [I(PGDEMOTE_KHUGEPAGED)] = "pgdemote_khugepaged",
1291 [I(PGDEMOTE_PROACTIVE)] = "pgdemote_proactive",
1292 [I(PGSTEAL_KSWAPD)] = "pgsteal_kswapd",
1293 [I(PGSTEAL_DIRECT)] = "pgsteal_direct",
1294 [I(PGSTEAL_KHUGEPAGED)] = "pgsteal_khugepaged",
1295 [I(PGSTEAL_PROACTIVE)] = "pgsteal_proactive",
1296 [I(PGSTEAL_ANON)] = "pgsteal_anon",
1297 [I(PGSTEAL_FILE)] = "pgsteal_file",
1298 [I(PGSCAN_KSWAPD)] = "pgscan_kswapd",
1299 [I(PGSCAN_DIRECT)] = "pgscan_direct",
1300 [I(PGSCAN_KHUGEPAGED)] = "pgscan_khugepaged",
1301 [I(PGSCAN_PROACTIVE)] = "pgscan_proactive",
1302 [I(PGSCAN_ANON)] = "pgscan_anon",
1303 [I(PGSCAN_FILE)] = "pgscan_file",
1304 [I(PGROTATE_ANON)] = "pgrotate_anon",
1305 [I(PGROTATE_FILE)] = "pgrotate_file",
1306 [I(PGREFILL)] = "pgrefill",
1307#ifdef CONFIG_HUGETLB_PAGE
1308 [I(NR_HUGETLB)] = "nr_hugetlb",
1309#endif
1310 [I(NR_BALLOON_PAGES)] = "nr_balloon_pages",
1311 [I(NR_KERNEL_FILE_PAGES)] = "nr_kernel_file_pages",
1312 [I(NR_GPU_ACTIVE)] = "nr_gpu_active",
1313 [I(NR_GPU_RECLAIM)] = "nr_gpu_reclaim",
1314#undef I
1315
1316 /* system-wide enum vm_stat_item counters */
1317#define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \
1318 NR_VM_NODE_STAT_ITEMS + x)
1319 [I(NR_DIRTY_THRESHOLD)] = "nr_dirty_threshold",
1320 [I(NR_DIRTY_BG_THRESHOLD)] = "nr_dirty_background_threshold",
1321 [I(NR_MEMMAP_PAGES)] = "nr_memmap_pages",
1322 [I(NR_MEMMAP_BOOT_PAGES)] = "nr_memmap_boot_pages",
1323#undef I
1324
1325#if defined(CONFIG_VM_EVENT_COUNTERS)
1326 /* enum vm_event_item counters */
1327#define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \
1328 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS + x)
1329
1330 [I(PGPGIN)] = "pgpgin",
1331 [I(PGPGOUT)] = "pgpgout",
1332 [I(PSWPIN)] = "pswpin",
1333 [I(PSWPOUT)] = "pswpout",
1334
1335#define OFF (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \
1336 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS)
1337 TEXTS_FOR_ZONES(OFF+PGALLOC, "pgalloc")
1338 TEXTS_FOR_ZONES(OFF+ALLOCSTALL, "allocstall")
1339 TEXTS_FOR_ZONES(OFF+PGSCAN_SKIP, "pgskip")
1340#undef OFF
1341
1342 [I(PGFREE)] = "pgfree",
1343 [I(PGACTIVATE)] = "pgactivate",
1344 [I(PGDEACTIVATE)] = "pgdeactivate",
1345 [I(PGLAZYFREE)] = "pglazyfree",
1346
1347 [I(PGFAULT)] = "pgfault",
1348 [I(PGMAJFAULT)] = "pgmajfault",
1349 [I(PGLAZYFREED)] = "pglazyfreed",
1350
1351 [I(PGREUSE)] = "pgreuse",
1352 [I(PGSCAN_DIRECT_THROTTLE)] = "pgscan_direct_throttle",
1353
1354#ifdef CONFIG_NUMA
1355 [I(PGSCAN_ZONE_RECLAIM_SUCCESS)] = "zone_reclaim_success",
1356 [I(PGSCAN_ZONE_RECLAIM_FAILED)] = "zone_reclaim_failed",
1357#endif
1358 [I(PGINODESTEAL)] = "pginodesteal",
1359 [I(SLABS_SCANNED)] = "slabs_scanned",
1360 [I(KSWAPD_INODESTEAL)] = "kswapd_inodesteal",
1361 [I(KSWAPD_LOW_WMARK_HIT_QUICKLY)] = "kswapd_low_wmark_hit_quickly",
1362 [I(KSWAPD_HIGH_WMARK_HIT_QUICKLY)] = "kswapd_high_wmark_hit_quickly",
1363 [I(PAGEOUTRUN)] = "pageoutrun",
1364
1365 [I(PGROTATED)] = "pgrotated",
1366
1367 [I(DROP_PAGECACHE)] = "drop_pagecache",
1368 [I(DROP_SLAB)] = "drop_slab",
1369 [I(OOM_KILL)] = "oom_kill",
1370
1371#ifdef CONFIG_NUMA_BALANCING
1372 [I(NUMA_PTE_UPDATES)] = "numa_pte_updates",
1373 [I(NUMA_HUGE_PTE_UPDATES)] = "numa_huge_pte_updates",
1374 [I(NUMA_HINT_FAULTS)] = "numa_hint_faults",
1375 [I(NUMA_HINT_FAULTS_LOCAL)] = "numa_hint_faults_local",
1376 [I(NUMA_PAGE_MIGRATE)] = "numa_pages_migrated",
1377#endif
1378#ifdef CONFIG_MIGRATION
1379 [I(PGMIGRATE_SUCCESS)] = "pgmigrate_success",
1380 [I(PGMIGRATE_FAIL)] = "pgmigrate_fail",
1381 [I(THP_MIGRATION_SUCCESS)] = "thp_migration_success",
1382 [I(THP_MIGRATION_FAIL)] = "thp_migration_fail",
1383 [I(THP_MIGRATION_SPLIT)] = "thp_migration_split",
1384#endif
1385#ifdef CONFIG_COMPACTION
1386 [I(COMPACTMIGRATE_SCANNED)] = "compact_migrate_scanned",
1387 [I(COMPACTFREE_SCANNED)] = "compact_free_scanned",
1388 [I(COMPACTISOLATED)] = "compact_isolated",
1389 [I(COMPACTSTALL)] = "compact_stall",
1390 [I(COMPACTFAIL)] = "compact_fail",
1391 [I(COMPACTSUCCESS)] = "compact_success",
1392 [I(KCOMPACTD_WAKE)] = "compact_daemon_wake",
1393 [I(KCOMPACTD_MIGRATE_SCANNED)] = "compact_daemon_migrate_scanned",
1394 [I(KCOMPACTD_FREE_SCANNED)] = "compact_daemon_free_scanned",
1395#endif
1396
1397#ifdef CONFIG_HUGETLB_PAGE
1398 [I(HTLB_BUDDY_PGALLOC)] = "htlb_buddy_alloc_success",
1399 [I(HTLB_BUDDY_PGALLOC_FAIL)] = "htlb_buddy_alloc_fail",
1400#endif
1401#ifdef CONFIG_CMA
1402 [I(CMA_ALLOC_SUCCESS)] = "cma_alloc_success",
1403 [I(CMA_ALLOC_FAIL)] = "cma_alloc_fail",
1404#endif
1405 [I(UNEVICTABLE_PGCULLED)] = "unevictable_pgs_culled",
1406 [I(UNEVICTABLE_PGSCANNED)] = "unevictable_pgs_scanned",
1407 [I(UNEVICTABLE_PGRESCUED)] = "unevictable_pgs_rescued",
1408 [I(UNEVICTABLE_PGMLOCKED)] = "unevictable_pgs_mlocked",
1409 [I(UNEVICTABLE_PGMUNLOCKED)] = "unevictable_pgs_munlocked",
1410 [I(UNEVICTABLE_PGCLEARED)] = "unevictable_pgs_cleared",
1411 [I(UNEVICTABLE_PGSTRANDED)] = "unevictable_pgs_stranded",
1412
1413#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1414 [I(THP_FAULT_ALLOC)] = "thp_fault_alloc",
1415 [I(THP_FAULT_FALLBACK)] = "thp_fault_fallback",
1416 [I(THP_FAULT_FALLBACK_CHARGE)] = "thp_fault_fallback_charge",
1417 [I(THP_COLLAPSE_ALLOC)] = "thp_collapse_alloc",
1418 [I(THP_COLLAPSE_ALLOC_FAILED)] = "thp_collapse_alloc_failed",
1419 [I(THP_FILE_ALLOC)] = "thp_file_alloc",
1420 [I(THP_FILE_FALLBACK)] = "thp_file_fallback",
1421 [I(THP_FILE_FALLBACK_CHARGE)] = "thp_file_fallback_charge",
1422 [I(THP_FILE_MAPPED)] = "thp_file_mapped",
1423 [I(THP_SPLIT_PAGE)] = "thp_split_page",
1424 [I(THP_SPLIT_PAGE_FAILED)] = "thp_split_page_failed",
1425 [I(THP_DEFERRED_SPLIT_PAGE)] = "thp_deferred_split_page",
1426 [I(THP_UNDERUSED_SPLIT_PAGE)] = "thp_underused_split_page",
1427 [I(THP_SPLIT_PMD)] = "thp_split_pmd",
1428 [I(THP_SCAN_EXCEED_NONE_PTE)] = "thp_scan_exceed_none_pte",
1429 [I(THP_SCAN_EXCEED_SWAP_PTE)] = "thp_scan_exceed_swap_pte",
1430 [I(THP_SCAN_EXCEED_SHARED_PTE)] = "thp_scan_exceed_share_pte",
1431#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1432 [I(THP_SPLIT_PUD)] = "thp_split_pud",
1433#endif
1434 [I(THP_ZERO_PAGE_ALLOC)] = "thp_zero_page_alloc",
1435 [I(THP_ZERO_PAGE_ALLOC_FAILED)] = "thp_zero_page_alloc_failed",
1436 [I(THP_SWPOUT)] = "thp_swpout",
1437 [I(THP_SWPOUT_FALLBACK)] = "thp_swpout_fallback",
1438#endif
1439#ifdef CONFIG_BALLOON
1440 [I(BALLOON_INFLATE)] = "balloon_inflate",
1441 [I(BALLOON_DEFLATE)] = "balloon_deflate",
1442#ifdef CONFIG_BALLOON_MIGRATION
1443 [I(BALLOON_MIGRATE)] = "balloon_migrate",
1444#endif /* CONFIG_BALLOON_MIGRATION */
1445#endif /* CONFIG_BALLOON */
1446#ifdef CONFIG_DEBUG_TLBFLUSH
1447 [I(NR_TLB_REMOTE_FLUSH)] = "nr_tlb_remote_flush",
1448 [I(NR_TLB_REMOTE_FLUSH_RECEIVED)] = "nr_tlb_remote_flush_received",
1449 [I(NR_TLB_LOCAL_FLUSH_ALL)] = "nr_tlb_local_flush_all",
1450 [I(NR_TLB_LOCAL_FLUSH_ONE)] = "nr_tlb_local_flush_one",
1451#endif /* CONFIG_DEBUG_TLBFLUSH */
1452
1453#ifdef CONFIG_SWAP
1454 [I(SWAP_RA)] = "swap_ra",
1455 [I(SWAP_RA_HIT)] = "swap_ra_hit",
1456 [I(SWPIN_ZERO)] = "swpin_zero",
1457 [I(SWPOUT_ZERO)] = "swpout_zero",
1458#ifdef CONFIG_KSM
1459 [I(KSM_SWPIN_COPY)] = "ksm_swpin_copy",
1460#endif
1461#endif
1462#ifdef CONFIG_KSM
1463 [I(COW_KSM)] = "cow_ksm",
1464#endif
1465#ifdef CONFIG_ZSWAP
1466 [I(ZSWPIN)] = "zswpin",
1467 [I(ZSWPOUT)] = "zswpout",
1468 [I(ZSWPWB)] = "zswpwb",
1469#endif
1470#ifdef CONFIG_X86
1471 [I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits",
1472 [I(DIRECT_MAP_LEVEL3_SPLIT)] = "direct_map_level3_splits",
1473 [I(DIRECT_MAP_LEVEL2_COLLAPSE)] = "direct_map_level2_collapses",
1474 [I(DIRECT_MAP_LEVEL3_COLLAPSE)] = "direct_map_level3_collapses",
1475#endif
1476#ifdef CONFIG_PER_VMA_LOCK_STATS
1477 [I(VMA_LOCK_SUCCESS)] = "vma_lock_success",
1478 [I(VMA_LOCK_ABORT)] = "vma_lock_abort",
1479 [I(VMA_LOCK_RETRY)] = "vma_lock_retry",
1480 [I(VMA_LOCK_MISS)] = "vma_lock_miss",
1481#endif
1482#ifdef CONFIG_DEBUG_STACK_USAGE
1483 [I(KSTACK_1K)] = "kstack_1k",
1484#if THREAD_SIZE > 1024
1485 [I(KSTACK_2K)] = "kstack_2k",
1486#endif
1487#if THREAD_SIZE > 2048
1488 [I(KSTACK_4K)] = "kstack_4k",
1489#endif
1490#if THREAD_SIZE > 4096
1491 [I(KSTACK_8K)] = "kstack_8k",
1492#endif
1493#if THREAD_SIZE > 8192
1494 [I(KSTACK_16K)] = "kstack_16k",
1495#endif
1496#if THREAD_SIZE > 16384
1497 [I(KSTACK_32K)] = "kstack_32k",
1498#endif
1499#if THREAD_SIZE > 32768
1500 [I(KSTACK_64K)] = "kstack_64k",
1501#endif
1502#if THREAD_SIZE > 65536
1503 [I(KSTACK_REST)] = "kstack_rest",
1504#endif
1505#endif /* CONFIG_DEBUG_STACK_USAGE */
1506#ifdef CONFIG_SWAP
1507 [I(NRSWPIN)] = "nrswpin",
1508 [I(NRSWPOUT)] = "nrswpout",
1509#endif /* CONFIG_SWAP */
1510#undef I
1511#endif /* CONFIG_VM_EVENT_COUNTERS */
1512};
1513#endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
1514
1515#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1516 defined(CONFIG_PROC_FS)
1517static void *frag_start(struct seq_file *m, loff_t *pos)
1518{
1519 pg_data_t *pgdat;
1520 loff_t node = *pos;
1521
1522 for (pgdat = first_online_pgdat();
1523 pgdat && node;
1524 pgdat = next_online_pgdat(pgdat))
1525 --node;
1526
1527 return pgdat;
1528}
1529
1530static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
1531{
1532 pg_data_t *pgdat = (pg_data_t *)arg;
1533
1534 (*pos)++;
1535 return next_online_pgdat(pgdat);
1536}
1537
1538static void frag_stop(struct seq_file *m, void *arg)
1539{
1540}
1541
1542/*
1543 * Walk zones in a node and print using a callback.
1544 * If @assert_populated is true, only use callback for zones that are populated.
1545 */
1546static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
1547 bool assert_populated, bool nolock,
1548 void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
1549{
1550 struct zone *zone;
1551 struct zone *node_zones = pgdat->node_zones;
1552 unsigned long flags;
1553
1554 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
1555 if (assert_populated && !populated_zone(zone))
1556 continue;
1557
1558 if (!nolock)
1559 spin_lock_irqsave(&zone->lock, flags);
1560 print(m, pgdat, zone);
1561 if (!nolock)
1562 spin_unlock_irqrestore(&zone->lock, flags);
1563 }
1564}
1565#endif
1566
1567#ifdef CONFIG_PROC_FS
1568static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
1569 struct zone *zone)
1570{
1571 int order;
1572
1573 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1574 for (order = 0; order < NR_PAGE_ORDERS; ++order)
1575 /*
1576 * Access to nr_free is lockless as nr_free is used only for
1577 * printing purposes. Use data_race to avoid KCSAN warning.
1578 */
1579 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));
1580 seq_putc(m, '\n');
1581}
1582
1583/*
1584 * This walks the free areas for each zone.
1585 */
1586static int frag_show(struct seq_file *m, void *arg)
1587{
1588 pg_data_t *pgdat = (pg_data_t *)arg;
1589 walk_zones_in_node(m, pgdat, true, true, frag_show_print);
1590 return 0;
1591}
1592
1593static void pagetypeinfo_showfree_print(struct seq_file *m,
1594 pg_data_t *pgdat, struct zone *zone)
1595{
1596 int order, mtype;
1597
1598 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
1599 seq_printf(m, "Node %4d, zone %8s, type %12s ",
1600 pgdat->node_id,
1601 zone->name,
1602 migratetype_names[mtype]);
1603 for (order = 0; order < NR_PAGE_ORDERS; ++order) {
1604 unsigned long freecount = 0;
1605 struct free_area *area;
1606 struct list_head *curr;
1607 bool overflow = false;
1608
1609 area = &(zone->free_area[order]);
1610
1611 list_for_each(curr, &area->free_list[mtype]) {
1612 /*
1613 * Cap the free_list iteration because it might
1614 * be really large and we are under a spinlock
1615 * so a long time spent here could trigger a
1616 * hard lockup detector. Anyway this is a
1617 * debugging tool so knowing there is a handful
1618 * of pages of this order should be more than
1619 * sufficient.
1620 */
1621 if (++freecount >= 100000) {
1622 overflow = true;
1623 break;
1624 }
1625 }
1626 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount);
1627 spin_unlock_irq(&zone->lock);
1628 cond_resched();
1629 spin_lock_irq(&zone->lock);
1630 }
1631 seq_putc(m, '\n');
1632 }
1633}
1634
1635/* Print out the free pages at each order for each migratetype */
1636static void pagetypeinfo_showfree(struct seq_file *m, void *arg)
1637{
1638 int order;
1639 pg_data_t *pgdat = (pg_data_t *)arg;
1640
1641 /* Print header */
1642 seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
1643 for (order = 0; order < NR_PAGE_ORDERS; ++order)
1644 seq_printf(m, "%6d ", order);
1645 seq_putc(m, '\n');
1646
1647 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print);
1648}
1649
1650static void pagetypeinfo_showblockcount_print(struct seq_file *m,
1651 pg_data_t *pgdat, struct zone *zone)
1652{
1653 int mtype;
1654 unsigned long pfn;
1655 unsigned long start_pfn = zone->zone_start_pfn;
1656 unsigned long end_pfn = zone_end_pfn(zone);
1657 unsigned long count[MIGRATE_TYPES] = { 0, };
1658
1659 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
1660 struct page *page;
1661
1662 page = pfn_to_online_page(pfn);
1663 if (!page)
1664 continue;
1665
1666 if (page_zone(page) != zone)
1667 continue;
1668
1669 mtype = get_pageblock_migratetype(page);
1670
1671 if (mtype < MIGRATE_TYPES)
1672 count[mtype]++;
1673 }
1674
1675 /* Print counts */
1676 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1677 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1678 seq_printf(m, "%12lu ", count[mtype]);
1679 seq_putc(m, '\n');
1680}
1681
1682/* Print out the number of pageblocks for each migratetype */
1683static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
1684{
1685 int mtype;
1686 pg_data_t *pgdat = (pg_data_t *)arg;
1687
1688 seq_printf(m, "\n%-23s", "Number of blocks type ");
1689 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1690 seq_printf(m, "%12s ", migratetype_names[mtype]);
1691 seq_putc(m, '\n');
1692 walk_zones_in_node(m, pgdat, true, false,
1693 pagetypeinfo_showblockcount_print);
1694}
1695
1696/*
1697 * Print out the number of pageblocks for each migratetype that contain pages
1698 * of other types. This gives an indication of how well fallbacks are being
1699 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1700 * to determine what is going on
1701 */
1702static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
1703{
1704#ifdef CONFIG_PAGE_OWNER
1705 int mtype;
1706
1707 if (!static_branch_unlikely(&page_owner_inited))
1708 return;
1709
1710 drain_all_pages(NULL);
1711
1712 seq_printf(m, "\n%-23s", "Number of mixed blocks ");
1713 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1714 seq_printf(m, "%12s ", migratetype_names[mtype]);
1715 seq_putc(m, '\n');
1716
1717 walk_zones_in_node(m, pgdat, true, true,
1718 pagetypeinfo_showmixedcount_print);
1719#endif /* CONFIG_PAGE_OWNER */
1720}
1721
1722/*
1723 * This prints out statistics in relation to grouping pages by mobility.
1724 * It is expensive to collect so do not constantly read the file.
1725 */
1726static int pagetypeinfo_show(struct seq_file *m, void *arg)
1727{
1728 pg_data_t *pgdat = (pg_data_t *)arg;
1729
1730 /* check memoryless node */
1731 if (!node_state(pgdat->node_id, N_MEMORY))
1732 return 0;
1733
1734 seq_printf(m, "Page block order: %d\n", pageblock_order);
1735 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
1736 seq_putc(m, '\n');
1737 pagetypeinfo_showfree(m, pgdat);
1738 pagetypeinfo_showblockcount(m, pgdat);
1739 pagetypeinfo_showmixedcount(m, pgdat);
1740
1741 return 0;
1742}
1743
1744static const struct seq_operations fragmentation_op = {
1745 .start = frag_start,
1746 .next = frag_next,
1747 .stop = frag_stop,
1748 .show = frag_show,
1749};
1750
1751static const struct seq_operations pagetypeinfo_op = {
1752 .start = frag_start,
1753 .next = frag_next,
1754 .stop = frag_stop,
1755 .show = pagetypeinfo_show,
1756};
1757
1758static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
1759{
1760 int zid;
1761
1762 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1763 struct zone *compare = &pgdat->node_zones[zid];
1764
1765 if (populated_zone(compare))
1766 return zone == compare;
1767 }
1768
1769 return false;
1770}
1771
1772static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1773 struct zone *zone)
1774{
1775 int i;
1776 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
1777 if (is_zone_first_populated(pgdat, zone)) {
1778 seq_printf(m, "\n per-node stats");
1779 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1780 unsigned long pages = node_page_state_pages(pgdat, i);
1781
1782 if (vmstat_item_print_in_thp(i))
1783 pages /= HPAGE_PMD_NR;
1784 seq_printf(m, "\n %-12s %lu", node_stat_name(i),
1785 pages);
1786 }
1787 }
1788 seq_printf(m,
1789 "\n pages free %lu"
1790 "\n boost %lu"
1791 "\n min %lu"
1792 "\n low %lu"
1793 "\n high %lu"
1794 "\n promo %lu"
1795 "\n spanned %lu"
1796 "\n present %lu"
1797 "\n managed %lu"
1798 "\n cma %lu",
1799 zone_page_state(zone, NR_FREE_PAGES),
1800 zone->watermark_boost,
1801 min_wmark_pages(zone),
1802 low_wmark_pages(zone),
1803 high_wmark_pages(zone),
1804 promo_wmark_pages(zone),
1805 zone->spanned_pages,
1806 zone->present_pages,
1807 zone_managed_pages(zone),
1808 zone_cma_pages(zone));
1809
1810 seq_printf(m,
1811 "\n protection: (%ld",
1812 zone->lowmem_reserve[0]);
1813 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
1814 seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
1815 seq_putc(m, ')');
1816
1817 /* If unpopulated, no other information is useful */
1818 if (!populated_zone(zone)) {
1819 seq_putc(m, '\n');
1820 return;
1821 }
1822
1823 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1824 seq_printf(m, "\n %-12s %lu", zone_stat_name(i),
1825 zone_page_state(zone, i));
1826
1827#ifdef CONFIG_NUMA
1828 fold_vm_zone_numa_events(zone);
1829 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
1830 seq_printf(m, "\n %-12s %lu", numa_stat_name(i),
1831 zone_numa_event_state(zone, i));
1832#endif
1833
1834 seq_printf(m, "\n pagesets");
1835 for_each_online_cpu(i) {
1836 struct per_cpu_pages *pcp;
1837 struct per_cpu_zonestat __maybe_unused *pzstats;
1838
1839 pcp = per_cpu_ptr(zone->per_cpu_pageset, i);
1840 seq_printf(m,
1841 "\n cpu: %i"
1842 "\n count: %i"
1843 "\n high: %i"
1844 "\n batch: %i"
1845 "\n high_min: %i"
1846 "\n high_max: %i",
1847 i,
1848 pcp->count,
1849 pcp->high,
1850 pcp->batch,
1851 pcp->high_min,
1852 pcp->high_max);
1853#ifdef CONFIG_SMP
1854 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);
1855 seq_printf(m, "\n vm stats threshold: %d",
1856 pzstats->stat_threshold);
1857#endif
1858 }
1859 seq_printf(m,
1860 "\n node_unreclaimable: %u"
1861 "\n start_pfn: %lu"
1862 "\n reserved_highatomic: %lu"
1863 "\n free_highatomic: %lu",
1864 kswapd_test_hopeless(pgdat),
1865 zone->zone_start_pfn,
1866 zone->nr_reserved_highatomic,
1867 zone->nr_free_highatomic);
1868 seq_putc(m, '\n');
1869}
1870
1871/*
1872 * Output information about zones in @pgdat. All zones are printed regardless
1873 * of whether they are populated or not: lowmem_reserve_ratio operates on the
1874 * set of all zones and userspace would not be aware of such zones if they are
1875 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio).
1876 */
1877static int zoneinfo_show(struct seq_file *m, void *arg)
1878{
1879 pg_data_t *pgdat = (pg_data_t *)arg;
1880 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print);
1881 return 0;
1882}
1883
1884static const struct seq_operations zoneinfo_op = {
1885 .start = frag_start, /* iterate over all zones. The same as in
1886 * fragmentation. */
1887 .next = frag_next,
1888 .stop = frag_stop,
1889 .show = zoneinfo_show,
1890};
1891
1892#define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
1893 NR_VM_NUMA_EVENT_ITEMS + \
1894 NR_VM_NODE_STAT_ITEMS + \
1895 NR_VM_STAT_ITEMS + \
1896 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
1897 NR_VM_EVENT_ITEMS : 0))
1898
1899static void *vmstat_start(struct seq_file *m, loff_t *pos)
1900{
1901 unsigned long *v;
1902 int i;
1903
1904 if (*pos >= NR_VMSTAT_ITEMS)
1905 return NULL;
1906
1907 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) != NR_VMSTAT_ITEMS);
1908 fold_vm_numa_events();
1909 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL);
1910 m->private = v;
1911 if (!v)
1912 return ERR_PTR(-ENOMEM);
1913 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1914 v[i] = global_zone_page_state(i);
1915 v += NR_VM_ZONE_STAT_ITEMS;
1916
1917#ifdef CONFIG_NUMA
1918 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++)
1919 v[i] = global_numa_event_state(i);
1920 v += NR_VM_NUMA_EVENT_ITEMS;
1921#endif
1922
1923 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1924 v[i] = global_node_page_state_pages(i);
1925 if (vmstat_item_print_in_thp(i))
1926 v[i] /= HPAGE_PMD_NR;
1927 }
1928 v += NR_VM_NODE_STAT_ITEMS;
1929
1930 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1931 v + NR_DIRTY_THRESHOLD);
1932 v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages);
1933 v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages);
1934 v += NR_VM_STAT_ITEMS;
1935
1936#ifdef CONFIG_VM_EVENT_COUNTERS
1937 all_vm_events(v);
1938 v[PGPGIN] /= 2; /* sectors -> kbytes */
1939 v[PGPGOUT] /= 2;
1940#endif
1941 return (unsigned long *)m->private + *pos;
1942}
1943
1944static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
1945{
1946 (*pos)++;
1947 if (*pos >= NR_VMSTAT_ITEMS)
1948 return NULL;
1949 return (unsigned long *)m->private + *pos;
1950}
1951
1952static int vmstat_show(struct seq_file *m, void *arg)
1953{
1954 unsigned long *l = arg;
1955 unsigned long off = l - (unsigned long *)m->private;
1956
1957 seq_puts(m, vmstat_text[off]);
1958 seq_put_decimal_ull(m, " ", *l);
1959 seq_putc(m, '\n');
1960
1961 if (off == NR_VMSTAT_ITEMS - 1) {
1962 /*
1963 * We've come to the end - add any deprecated counters to avoid
1964 * breaking userspace which might depend on them being present.
1965 */
1966 seq_puts(m, "nr_unstable 0\n");
1967 }
1968 return 0;
1969}
1970
1971static void vmstat_stop(struct seq_file *m, void *arg)
1972{
1973 kfree(m->private);
1974 m->private = NULL;
1975}
1976
1977static const struct seq_operations vmstat_op = {
1978 .start = vmstat_start,
1979 .next = vmstat_next,
1980 .stop = vmstat_stop,
1981 .show = vmstat_show,
1982};
1983#endif /* CONFIG_PROC_FS */
1984
1985#ifdef CONFIG_SMP
1986static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
1987static int sysctl_stat_interval __read_mostly = HZ;
1988static int vmstat_late_init_done;
1989
1990#ifdef CONFIG_PROC_FS
1991static void refresh_vm_stats(struct work_struct *work)
1992{
1993 refresh_cpu_vm_stats(true);
1994}
1995
1996static int vmstat_refresh(const struct ctl_table *table, int write,
1997 void *buffer, size_t *lenp, loff_t *ppos)
1998{
1999 long val;
2000 int err;
2001 int i;
2002
2003 /*
2004 * The regular update, every sysctl_stat_interval, may come later
2005 * than expected: leaving a significant amount in per_cpu buckets.
2006 * This is particularly misleading when checking a quantity of HUGE
2007 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
2008 * which can equally be echo'ed to or cat'ted from (by root),
2009 * can be used to update the stats just before reading them.
2010 *
2011 * Oh, and since global_zone_page_state() etc. are so careful to hide
2012 * transiently negative values, report an error here if any of
2013 * the stats is negative, so we know to go looking for imbalance.
2014 */
2015 err = schedule_on_each_cpu(refresh_vm_stats);
2016 if (err)
2017 return err;
2018 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
2019 /*
2020 * Skip checking stats known to go negative occasionally.
2021 */
2022 switch (i) {
2023 case NR_ZONE_WRITE_PENDING:
2024 case NR_FREE_CMA_PAGES:
2025 continue;
2026 }
2027 val = atomic_long_read(&vm_zone_stat[i]);
2028 if (val < 0) {
2029 pr_warn("%s: %s %ld\n",
2030 __func__, zone_stat_name(i), val);
2031 }
2032 }
2033 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
2034 /*
2035 * Skip checking stats known to go negative occasionally.
2036 */
2037 switch (i) {
2038 case NR_WRITEBACK:
2039 continue;
2040 }
2041 val = atomic_long_read(&vm_node_stat[i]);
2042 if (val < 0) {
2043 pr_warn("%s: %s %ld\n",
2044 __func__, node_stat_name(i), val);
2045 }
2046 }
2047 if (write)
2048 *ppos += *lenp;
2049 else
2050 *lenp = 0;
2051 return 0;
2052}
2053#endif /* CONFIG_PROC_FS */
2054
2055static void vmstat_update(struct work_struct *w)
2056{
2057 if (refresh_cpu_vm_stats(true)) {
2058 /*
2059 * Counters were updated so we expect more updates
2060 * to occur in the future. Keep on running the
2061 * update worker thread.
2062 */
2063 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq,
2064 this_cpu_ptr(&vmstat_work),
2065 round_jiffies_relative(sysctl_stat_interval));
2066 }
2067}
2068
2069/*
2070 * Check if the diffs for a certain cpu indicate that
2071 * an update is needed.
2072 */
2073static bool need_update(int cpu)
2074{
2075 pg_data_t *last_pgdat = NULL;
2076 struct zone *zone;
2077
2078 for_each_populated_zone(zone) {
2079 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
2080 struct per_cpu_nodestat *n;
2081
2082 /*
2083 * The fast way of checking if there are any vmstat diffs.
2084 */
2085 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff)))
2086 return true;
2087
2088 if (last_pgdat == zone->zone_pgdat)
2089 continue;
2090 last_pgdat = zone->zone_pgdat;
2091 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);
2092 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff)))
2093 return true;
2094 }
2095 return false;
2096}
2097
2098/*
2099 * Switch off vmstat processing and then fold all the remaining differentials
2100 * until the diffs stay at zero. The function is used by NOHZ and can only be
2101 * invoked when tick processing is not active.
2102 */
2103void quiet_vmstat(void)
2104{
2105 if (system_state != SYSTEM_RUNNING)
2106 return;
2107
2108 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
2109 return;
2110
2111 if (!need_update(smp_processor_id()))
2112 return;
2113
2114 /*
2115 * Just refresh counters and do not care about the pending delayed
2116 * vmstat_update. It doesn't fire that often to matter and canceling
2117 * it would be too expensive from this path.
2118 * vmstat_shepherd will take care about that for us.
2119 */
2120 refresh_cpu_vm_stats(false);
2121}
2122
2123/*
2124 * Shepherd worker thread that checks the
2125 * differentials of processors that have their worker
2126 * threads for vm statistics updates disabled because of
2127 * inactivity.
2128 */
2129static void vmstat_shepherd(struct work_struct *w);
2130
2131static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
2132
2133void vmstat_flush_workqueue(void)
2134{
2135 flush_workqueue(mm_percpu_wq);
2136}
2137
2138static void vmstat_shepherd(struct work_struct *w)
2139{
2140 int cpu;
2141
2142 cpus_read_lock();
2143 /* Check processors whose vmstat worker threads have been disabled */
2144 for_each_online_cpu(cpu) {
2145 struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
2146
2147 /*
2148 * In kernel users of vmstat counters either require the precise value and
2149 * they are using zone_page_state_snapshot interface or they can live with
2150 * an imprecision as the regular flushing can happen at arbitrary time and
2151 * cumulative error can grow (see calculate_normal_threshold).
2152 *
2153 * From that POV the regular flushing can be postponed for CPUs that have
2154 * been isolated from the kernel interference without critical
2155 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd
2156 * for all isolated CPUs to avoid interference with the isolated workload.
2157 */
2158 scoped_guard(rcu) {
2159 if (cpu_is_isolated(cpu))
2160 continue;
2161
2162 if (!work_busy(&dw->work) && need_update(cpu))
2163 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0);
2164 }
2165
2166 cond_resched();
2167 }
2168 cpus_read_unlock();
2169
2170 schedule_delayed_work(&shepherd,
2171 round_jiffies_relative(sysctl_stat_interval));
2172}
2173
2174static void __init start_shepherd_timer(void)
2175{
2176 int cpu;
2177
2178 for_each_possible_cpu(cpu) {
2179 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
2180 vmstat_update);
2181
2182 /*
2183 * For secondary CPUs during CPU hotplug scenarios,
2184 * vmstat_cpu_online() will enable the work.
2185 * mm/vmstat:online enables and disables vmstat_work
2186 * symmetrically during CPU hotplug events.
2187 */
2188 if (!cpu_online(cpu))
2189 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu));
2190 }
2191
2192 schedule_delayed_work(&shepherd,
2193 round_jiffies_relative(sysctl_stat_interval));
2194}
2195
2196static void __init init_cpu_node_state(void)
2197{
2198 int node;
2199
2200 for_each_online_node(node) {
2201 if (!cpumask_empty(cpumask_of_node(node)))
2202 node_set_state(node, N_CPU);
2203 }
2204}
2205
2206static int vmstat_cpu_online(unsigned int cpu)
2207{
2208 if (vmstat_late_init_done)
2209 refresh_zone_stat_thresholds();
2210
2211 if (!node_state(cpu_to_node(cpu), N_CPU)) {
2212 node_set_state(cpu_to_node(cpu), N_CPU);
2213 }
2214 enable_delayed_work(&per_cpu(vmstat_work, cpu));
2215
2216 return 0;
2217}
2218
2219static int vmstat_cpu_down_prep(unsigned int cpu)
2220{
2221 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu));
2222 return 0;
2223}
2224
2225static int vmstat_cpu_dead(unsigned int cpu)
2226{
2227 const struct cpumask *node_cpus;
2228 int node;
2229
2230 node = cpu_to_node(cpu);
2231
2232 refresh_zone_stat_thresholds();
2233 node_cpus = cpumask_of_node(node);
2234 if (!cpumask_empty(node_cpus))
2235 return 0;
2236
2237 node_clear_state(node, N_CPU);
2238
2239 return 0;
2240}
2241
2242static int __init vmstat_late_init(void)
2243{
2244 refresh_zone_stat_thresholds();
2245 vmstat_late_init_done = 1;
2246
2247 return 0;
2248}
2249late_initcall(vmstat_late_init);
2250#endif
2251
2252#ifdef CONFIG_PROC_FS
2253static const struct ctl_table vmstat_table[] = {
2254#ifdef CONFIG_SMP
2255 {
2256 .procname = "stat_interval",
2257 .data = &sysctl_stat_interval,
2258 .maxlen = sizeof(sysctl_stat_interval),
2259 .mode = 0644,
2260 .proc_handler = proc_dointvec_jiffies,
2261 },
2262 {
2263 .procname = "stat_refresh",
2264 .data = NULL,
2265 .maxlen = 0,
2266 .mode = 0600,
2267 .proc_handler = vmstat_refresh,
2268 },
2269#endif
2270#ifdef CONFIG_NUMA
2271 {
2272 .procname = "numa_stat",
2273 .data = &sysctl_vm_numa_stat,
2274 .maxlen = sizeof(int),
2275 .mode = 0644,
2276 .proc_handler = sysctl_vm_numa_stat_handler,
2277 .extra1 = SYSCTL_ZERO,
2278 .extra2 = SYSCTL_ONE,
2279 },
2280#endif
2281};
2282#endif
2283
2284struct workqueue_struct *mm_percpu_wq;
2285
2286void __init init_mm_internals(void)
2287{
2288 int ret __maybe_unused;
2289
2290 mm_percpu_wq = alloc_workqueue("mm_percpu_wq",
2291 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2292
2293#ifdef CONFIG_SMP
2294 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead",
2295 NULL, vmstat_cpu_dead);
2296 if (ret < 0)
2297 pr_err("vmstat: failed to register 'dead' hotplug state\n");
2298
2299 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online",
2300 vmstat_cpu_online,
2301 vmstat_cpu_down_prep);
2302 if (ret < 0)
2303 pr_err("vmstat: failed to register 'online' hotplug state\n");
2304
2305 cpus_read_lock();
2306 init_cpu_node_state();
2307 cpus_read_unlock();
2308
2309 start_shepherd_timer();
2310#endif
2311#ifdef CONFIG_PROC_FS
2312 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op);
2313 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op);
2314 proc_create_seq("vmstat", 0444, NULL, &vmstat_op);
2315 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op);
2316 register_sysctl_init("vm", vmstat_table);
2317#endif
2318}
2319
2320#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2321
2322/*
2323 * Return an index indicating how much of the available free memory is
2324 * unusable for an allocation of the requested size.
2325 */
2326static int unusable_free_index(unsigned int order,
2327 struct contig_page_info *info)
2328{
2329 /* No free memory is interpreted as all free memory is unusable */
2330 if (info->free_pages == 0)
2331 return 1000;
2332
2333 /*
2334 * Index should be a value between 0 and 1. Return a value to 3
2335 * decimal places.
2336 *
2337 * 0 => no fragmentation
2338 * 1 => high fragmentation
2339 */
2340 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
2341
2342}
2343
2344static void unusable_show_print(struct seq_file *m,
2345 pg_data_t *pgdat, struct zone *zone)
2346{
2347 unsigned int order;
2348 int index;
2349 struct contig_page_info info;
2350
2351 seq_printf(m, "Node %d, zone %8s ",
2352 pgdat->node_id,
2353 zone->name);
2354 for (order = 0; order < NR_PAGE_ORDERS; ++order) {
2355 fill_contig_page_info(zone, order, &info);
2356 index = unusable_free_index(order, &info);
2357 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2358 }
2359
2360 seq_putc(m, '\n');
2361}
2362
2363/*
2364 * Display unusable free space index
2365 *
2366 * The unusable free space index measures how much of the available free
2367 * memory cannot be used to satisfy an allocation of a given size and is a
2368 * value between 0 and 1. The higher the value, the more of free memory is
2369 * unusable and by implication, the worse the external fragmentation is. This
2370 * can be expressed as a percentage by multiplying by 100.
2371 */
2372static int unusable_show(struct seq_file *m, void *arg)
2373{
2374 pg_data_t *pgdat = (pg_data_t *)arg;
2375
2376 /* check memoryless node */
2377 if (!node_state(pgdat->node_id, N_MEMORY))
2378 return 0;
2379
2380 walk_zones_in_node(m, pgdat, true, false, unusable_show_print);
2381
2382 return 0;
2383}
2384
2385static const struct seq_operations unusable_sops = {
2386 .start = frag_start,
2387 .next = frag_next,
2388 .stop = frag_stop,
2389 .show = unusable_show,
2390};
2391
2392DEFINE_SEQ_ATTRIBUTE(unusable);
2393
2394static void extfrag_show_print(struct seq_file *m,
2395 pg_data_t *pgdat, struct zone *zone)
2396{
2397 unsigned int order;
2398 int index;
2399
2400 /* Alloc on stack as interrupts are disabled for zone walk */
2401 struct contig_page_info info;
2402
2403 seq_printf(m, "Node %d, zone %8s ",
2404 pgdat->node_id,
2405 zone->name);
2406 for (order = 0; order < NR_PAGE_ORDERS; ++order) {
2407 fill_contig_page_info(zone, order, &info);
2408 index = __fragmentation_index(order, &info);
2409 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000);
2410 }
2411
2412 seq_putc(m, '\n');
2413}
2414
2415/*
2416 * Display fragmentation index for orders that allocations would fail for
2417 */
2418static int extfrag_show(struct seq_file *m, void *arg)
2419{
2420 pg_data_t *pgdat = (pg_data_t *)arg;
2421
2422 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print);
2423
2424 return 0;
2425}
2426
2427static const struct seq_operations extfrag_sops = {
2428 .start = frag_start,
2429 .next = frag_next,
2430 .stop = frag_stop,
2431 .show = extfrag_show,
2432};
2433
2434DEFINE_SEQ_ATTRIBUTE(extfrag);
2435
2436static int __init extfrag_debug_init(void)
2437{
2438 struct dentry *extfrag_debug_root;
2439
2440 extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
2441
2442 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL,
2443 &unusable_fops);
2444
2445 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL,
2446 &extfrag_fops);
2447
2448 return 0;
2449}
2450
2451module_init(extfrag_debug_init);
2452
2453#endif