projects/linux·blob master · refs

3631 lines · 88.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/pagewalk.h>
3#include <linux/mm_inline.h>
4#include <linux/hugetlb.h>
5#include <linux/huge_mm.h>
6#include <linux/mount.h>
7#include <linux/ksm.h>
8#include <linux/seq_file.h>
9#include <linux/highmem.h>
10#include <linux/ptrace.h>
11#include <linux/slab.h>
12#include <linux/pagemap.h>
13#include <linux/mempolicy.h>
14#include <linux/rmap.h>
15#include <linux/swap.h>
16#include <linux/sched/mm.h>
17#include <linux/leafops.h>
18#include <linux/mmu_notifier.h>
19#include <linux/page_idle.h>
20#include <linux/shmem_fs.h>
21#include <linux/uaccess.h>
22#include <linux/pkeys.h>
23#include <linux/minmax.h>
24#include <linux/overflow.h>
25#include <linux/buildid.h>
26
27#include <asm/elf.h>
28#include <asm/tlb.h>
29#include <asm/tlbflush.h>
30#include "internal.h"
31
32#define SENTINEL_VMA_END -1
33#define SENTINEL_VMA_GATE -2
34
35#define SEQ_PUT_DEC(str, val) \
36 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
37void task_mem(struct seq_file *m, struct mm_struct *mm)
38{
39 unsigned long text, lib, swap, anon, file, shmem;
40 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
41
42 anon = get_mm_counter_sum(mm, MM_ANONPAGES);
43 file = get_mm_counter_sum(mm, MM_FILEPAGES);
44 shmem = get_mm_counter_sum(mm, MM_SHMEMPAGES);
45
46 /*
47 * Note: to minimize their overhead, mm maintains hiwater_vm and
48 * hiwater_rss only when about to *lower* total_vm or rss. Any
49 * collector of these hiwater stats must therefore get total_vm
50 * and rss too, which will usually be the higher. Barriers? not
51 * worth the effort, such snapshots can always be inconsistent.
52 */
53 hiwater_vm = total_vm = mm->total_vm;
54 if (hiwater_vm < mm->hiwater_vm)
55 hiwater_vm = mm->hiwater_vm;
56 hiwater_rss = total_rss = anon + file + shmem;
57 if (hiwater_rss < mm->hiwater_rss)
58 hiwater_rss = mm->hiwater_rss;
59
60 /* split executable areas between text and lib */
61 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
62 text = min(text, mm->exec_vm << PAGE_SHIFT);
63 lib = (mm->exec_vm << PAGE_SHIFT) - text;
64
65 swap = get_mm_counter_sum(mm, MM_SWAPENTS);
66 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
67 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
68 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
69 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
70 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
71 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
72 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
73 SEQ_PUT_DEC(" kB\nRssFile:\t", file);
74 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
75 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
76 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
77 seq_put_decimal_ull_width(m,
78 " kB\nVmExe:\t", text >> 10, 8);
79 seq_put_decimal_ull_width(m,
80 " kB\nVmLib:\t", lib >> 10, 8);
81 seq_put_decimal_ull_width(m,
82 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
83 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
84 seq_puts(m, " kB\n");
85 hugetlb_report_usage(m, mm);
86}
87#undef SEQ_PUT_DEC
88
89unsigned long task_vsize(struct mm_struct *mm)
90{
91 return PAGE_SIZE * mm->total_vm;
92}
93
94unsigned long task_statm(struct mm_struct *mm,
95 unsigned long *shared, unsigned long *text,
96 unsigned long *data, unsigned long *resident)
97{
98 *shared = get_mm_counter_sum(mm, MM_FILEPAGES) +
99 get_mm_counter_sum(mm, MM_SHMEMPAGES);
100 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
101 >> PAGE_SHIFT;
102 *data = mm->data_vm + mm->stack_vm;
103 *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES);
104 return mm->total_vm;
105}
106
107#ifdef CONFIG_NUMA
108/*
109 * Save get_task_policy() for show_numa_map().
110 */
111static void hold_task_mempolicy(struct proc_maps_private *priv)
112{
113 struct task_struct *task = priv->task;
114
115 task_lock(task);
116 priv->task_mempolicy = get_task_policy(task);
117 mpol_get(priv->task_mempolicy);
118 task_unlock(task);
119}
120static void release_task_mempolicy(struct proc_maps_private *priv)
121{
122 mpol_put(priv->task_mempolicy);
123}
124#else
125static void hold_task_mempolicy(struct proc_maps_private *priv)
126{
127}
128static void release_task_mempolicy(struct proc_maps_private *priv)
129{
130}
131#endif
132
133#ifdef CONFIG_PER_VMA_LOCK
134
135static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
136{
137 int ret = mmap_read_lock_killable(lock_ctx->mm);
138
139 if (!ret)
140 lock_ctx->mmap_locked = true;
141
142 return ret;
143}
144
145static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
146{
147 mmap_read_unlock(lock_ctx->mm);
148 lock_ctx->mmap_locked = false;
149}
150
151static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx)
152{
153 lock_ctx->locked_vma = NULL;
154 lock_ctx->mmap_locked = false;
155}
156
157static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx)
158{
159 if (lock_ctx->locked_vma) {
160 vma_end_read(lock_ctx->locked_vma);
161 lock_ctx->locked_vma = NULL;
162 }
163}
164
165static inline bool lock_vma_range(struct seq_file *m,
166 struct proc_maps_locking_ctx *lock_ctx)
167{
168 rcu_read_lock();
169 reset_lock_ctx(lock_ctx);
170
171 return true;
172}
173
174static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx)
175{
176 if (lock_ctx->mmap_locked) {
177 unlock_ctx_mm(lock_ctx);
178 } else {
179 unlock_ctx_vma(lock_ctx);
180 rcu_read_unlock();
181 }
182}
183
184static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
185 loff_t last_pos)
186{
187 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx;
188 struct vm_area_struct *vma;
189
190 if (lock_ctx->mmap_locked)
191 return vma_next(&priv->iter);
192
193 unlock_ctx_vma(lock_ctx);
194 vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos);
195 if (!IS_ERR_OR_NULL(vma))
196 lock_ctx->locked_vma = vma;
197
198 return vma;
199}
200
201static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
202 loff_t pos)
203{
204 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx;
205
206 if (lock_ctx->mmap_locked)
207 return false;
208
209 rcu_read_unlock();
210 mmap_read_lock(lock_ctx->mm);
211 /* Reinitialize the iterator after taking mmap_lock */
212 vma_iter_set(&priv->iter, pos);
213 lock_ctx->mmap_locked = true;
214
215 return true;
216}
217
218static inline void drop_rcu(struct proc_maps_private *priv)
219{
220 if (priv->lock_ctx.mmap_locked)
221 return;
222
223 rcu_read_unlock();
224}
225
226static inline void reacquire_rcu(struct proc_maps_private *priv)
227{
228 if (priv->lock_ctx.mmap_locked)
229 return;
230
231 rcu_read_lock();
232 /* Reinitialize the iterator. */
233 vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end);
234}
235
236#else /* CONFIG_PER_VMA_LOCK */
237
238static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
239{
240 return mmap_read_lock_killable(lock_ctx->mm);
241}
242
243static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
244{
245 mmap_read_unlock(lock_ctx->mm);
246}
247
248static inline bool lock_vma_range(struct seq_file *m,
249 struct proc_maps_locking_ctx *lock_ctx)
250{
251 return lock_ctx_mm(lock_ctx) == 0;
252}
253
254static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx)
255{
256 unlock_ctx_mm(lock_ctx);
257}
258
259static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
260 loff_t last_pos)
261{
262 return vma_next(&priv->iter);
263}
264
265static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
266 loff_t pos)
267{
268 return false;
269}
270
271static inline void drop_rcu(struct proc_maps_private *priv) {}
272static inline void reacquire_rcu(struct proc_maps_private *priv) {}
273
274#endif /* CONFIG_PER_VMA_LOCK */
275
276static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos)
277{
278 struct proc_maps_private *priv = m->private;
279 struct vm_area_struct *vma;
280
281retry:
282 vma = get_next_vma(priv, *ppos);
283 /* EINTR of EAGAIN is possible */
284 if (IS_ERR(vma)) {
285 if (PTR_ERR(vma) == -EAGAIN && fallback_to_mmap_lock(priv, *ppos))
286 goto retry;
287
288 return vma;
289 }
290
291 /* Store previous position to be able to restart if needed */
292 priv->last_pos = *ppos;
293 if (vma) {
294 /*
295 * Track the end of the reported vma to ensure position changes
296 * even if previous vma was merged with the next vma and we
297 * found the extended vma with the same vm_start.
298 */
299 *ppos = vma->vm_end;
300 } else {
301 *ppos = SENTINEL_VMA_GATE;
302 vma = get_gate_vma(priv->lock_ctx.mm);
303 }
304
305 return vma;
306}
307
308static void *m_start(struct seq_file *m, loff_t *ppos)
309{
310 struct proc_maps_private *priv = m->private;
311 struct proc_maps_locking_ctx *lock_ctx;
312 loff_t last_addr = *ppos;
313 struct mm_struct *mm;
314
315 /* See m_next(). Zero at the start or after lseek. */
316 if (last_addr == SENTINEL_VMA_END)
317 return NULL;
318
319 priv->task = get_proc_task(priv->inode);
320 if (!priv->task)
321 return ERR_PTR(-ESRCH);
322
323 lock_ctx = &priv->lock_ctx;
324 mm = lock_ctx->mm;
325 if (!mm || !mmget_not_zero(mm)) {
326 put_task_struct(priv->task);
327 priv->task = NULL;
328 return NULL;
329 }
330
331 if (!lock_vma_range(m, lock_ctx)) {
332 mmput(mm);
333 put_task_struct(priv->task);
334 priv->task = NULL;
335 return ERR_PTR(-EINTR);
336 }
337
338 /*
339 * Reset current position if last_addr was set before
340 * and it's not a sentinel.
341 */
342 if (last_addr > 0)
343 *ppos = last_addr = priv->last_pos;
344 vma_iter_init(&priv->iter, mm, (unsigned long)last_addr);
345 hold_task_mempolicy(priv);
346 if (last_addr == SENTINEL_VMA_GATE)
347 return get_gate_vma(mm);
348
349 return proc_get_vma(m, ppos);
350}
351
352static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
353{
354 if (*ppos == SENTINEL_VMA_GATE) {
355 *ppos = SENTINEL_VMA_END;
356 return NULL;
357 }
358 return proc_get_vma(m, ppos);
359}
360
361static void m_stop(struct seq_file *m, void *v)
362{
363 struct proc_maps_private *priv = m->private;
364 struct mm_struct *mm = priv->lock_ctx.mm;
365
366 if (!priv->task)
367 return;
368
369 release_task_mempolicy(priv);
370 unlock_vma_range(&priv->lock_ctx);
371 mmput(mm);
372 put_task_struct(priv->task);
373 priv->task = NULL;
374}
375
376static int proc_maps_open(struct inode *inode, struct file *file,
377 const struct seq_operations *ops, int psize)
378{
379 struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
380
381 if (!priv)
382 return -ENOMEM;
383
384 priv->inode = inode;
385 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ);
386 if (IS_ERR(priv->lock_ctx.mm)) {
387 int err = PTR_ERR(priv->lock_ctx.mm);
388
389 seq_release_private(inode, file);
390 return err;
391 }
392
393 return 0;
394}
395
396static int proc_map_release(struct inode *inode, struct file *file)
397{
398 struct seq_file *seq = file->private_data;
399 struct proc_maps_private *priv = seq->private;
400
401 if (priv->lock_ctx.mm)
402 mmdrop(priv->lock_ctx.mm);
403
404 return seq_release_private(inode, file);
405}
406
407static int do_maps_open(struct inode *inode, struct file *file,
408 const struct seq_operations *ops)
409{
410 return proc_maps_open(inode, file, ops,
411 sizeof(struct proc_maps_private));
412}
413
414static void get_vma_name(struct vm_area_struct *vma,
415 const struct path **path,
416 const char **name,
417 const char **name_fmt)
418{
419 struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL;
420
421 *name = NULL;
422 *path = NULL;
423 *name_fmt = NULL;
424
425 /*
426 * Print the dentry name for named mappings, and a
427 * special [heap] marker for the heap:
428 */
429 if (vma->vm_file) {
430 /*
431 * If user named this anon shared memory via
432 * prctl(PR_SET_VMA ..., use the provided name.
433 */
434 if (anon_name) {
435 *name_fmt = "[anon_shmem:%s]";
436 *name = anon_name->name;
437 } else {
438 *path = file_user_path(vma->vm_file);
439 }
440 return;
441 }
442
443 if (vma->vm_ops && vma->vm_ops->name) {
444 *name = vma->vm_ops->name(vma);
445 if (*name)
446 return;
447 }
448
449 *name = arch_vma_name(vma);
450 if (*name)
451 return;
452
453 if (!vma->vm_mm) {
454 *name = "[vdso]";
455 return;
456 }
457
458 if (vma_is_initial_heap(vma)) {
459 *name = "[heap]";
460 return;
461 }
462
463 if (vma_is_initial_stack(vma)) {
464 *name = "[stack]";
465 return;
466 }
467
468 if (anon_name) {
469 *name_fmt = "[anon:%s]";
470 *name = anon_name->name;
471 return;
472 }
473}
474
475static void show_vma_header_prefix(struct seq_file *m,
476 unsigned long start, unsigned long end,
477 vm_flags_t flags, unsigned long long pgoff,
478 dev_t dev, u64 ino)
479{
480 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
481 seq_put_hex_ll(m, NULL, start, 8);
482 seq_put_hex_ll(m, "-", end, 8);
483 seq_putc(m, ' ');
484 seq_putc(m, flags & VM_READ ? 'r' : '-');
485 seq_putc(m, flags & VM_WRITE ? 'w' : '-');
486 seq_putc(m, flags & VM_EXEC ? 'x' : '-');
487 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
488 seq_put_hex_ll(m, " ", pgoff, 8);
489 seq_put_hex_ll(m, " ", MAJOR(dev), 2);
490 seq_put_hex_ll(m, ":", MINOR(dev), 2);
491 seq_put_decimal_ull(m, " ", ino);
492 seq_putc(m, ' ');
493}
494
495static void
496show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
497{
498 const struct path *path;
499 const char *name_fmt, *name;
500 vm_flags_t flags = vma->vm_flags;
501 u64 ino = 0;
502 unsigned long long pgoff = 0;
503 unsigned long start, end;
504 dev_t dev = 0;
505
506 if (vma->vm_file) {
507 const struct inode *inode = file_user_inode(vma->vm_file);
508
509 dev = inode->i_sb->s_dev;
510 ino = inode->i_ino;
511 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
512 }
513
514 start = vma->vm_start;
515 end = vma->vm_end;
516 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
517
518 get_vma_name(vma, &path, &name, &name_fmt);
519 if (path) {
520 seq_pad(m, ' ');
521 seq_path(m, path, "\n");
522 } else if (name_fmt) {
523 seq_pad(m, ' ');
524 seq_printf(m, name_fmt, name);
525 } else if (name) {
526 seq_pad(m, ' ');
527 seq_puts(m, name);
528 }
529 seq_putc(m, '\n');
530}
531
532static int show_map(struct seq_file *m, void *v)
533{
534 show_map_vma(m, v);
535 return 0;
536}
537
538static const struct seq_operations proc_pid_maps_op = {
539 .start = m_start,
540 .next = m_next,
541 .stop = m_stop,
542 .show = show_map
543};
544
545static int pid_maps_open(struct inode *inode, struct file *file)
546{
547 return do_maps_open(inode, file, &proc_pid_maps_op);
548}
549
550#define PROCMAP_QUERY_VMA_FLAGS ( \
551 PROCMAP_QUERY_VMA_READABLE | \
552 PROCMAP_QUERY_VMA_WRITABLE | \
553 PROCMAP_QUERY_VMA_EXECUTABLE | \
554 PROCMAP_QUERY_VMA_SHARED \
555)
556
557#define PROCMAP_QUERY_VALID_FLAGS_MASK ( \
558 PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \
559 PROCMAP_QUERY_FILE_BACKED_VMA | \
560 PROCMAP_QUERY_VMA_FLAGS \
561)
562
563#ifdef CONFIG_PER_VMA_LOCK
564
565static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)
566{
567 reset_lock_ctx(lock_ctx);
568
569 return 0;
570}
571
572static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx)
573{
574 if (lock_ctx->mmap_locked)
575 unlock_ctx_mm(lock_ctx);
576 else
577 unlock_ctx_vma(lock_ctx);
578}
579
580static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,
581 unsigned long addr)
582{
583 struct mm_struct *mm = lock_ctx->mm;
584 struct vm_area_struct *vma;
585 struct vma_iterator vmi;
586
587 if (lock_ctx->mmap_locked)
588 return find_vma(mm, addr);
589
590 /* Unlock previously locked VMA and find the next one under RCU */
591 unlock_ctx_vma(lock_ctx);
592 rcu_read_lock();
593 vma_iter_init(&vmi, mm, addr);
594 vma = lock_next_vma(mm, &vmi, addr);
595 rcu_read_unlock();
596
597 if (!vma)
598 return NULL;
599
600 if (!IS_ERR(vma)) {
601 lock_ctx->locked_vma = vma;
602 return vma;
603 }
604
605 if (PTR_ERR(vma) == -EAGAIN) {
606 /* Fallback to mmap_lock on vma->vm_refcnt overflow */
607 mmap_read_lock(mm);
608 vma = find_vma(mm, addr);
609 lock_ctx->mmap_locked = true;
610 }
611
612 return vma;
613}
614
615#else /* CONFIG_PER_VMA_LOCK */
616
617static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)
618{
619 return mmap_read_lock_killable(lock_ctx->mm);
620}
621
622static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx)
623{
624 mmap_read_unlock(lock_ctx->mm);
625}
626
627static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,
628 unsigned long addr)
629{
630 return find_vma(lock_ctx->mm, addr);
631}
632
633#endif /* CONFIG_PER_VMA_LOCK */
634
635static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx,
636 unsigned long addr, u32 flags)
637{
638 struct vm_area_struct *vma;
639
640next_vma:
641 vma = query_vma_find_by_addr(lock_ctx, addr);
642 if (IS_ERR(vma))
643 return vma;
644
645 if (!vma)
646 goto no_vma;
647
648 /* user requested only file-backed VMA, keep iterating */
649 if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file)
650 goto skip_vma;
651
652 /* VMA permissions should satisfy query flags */
653 if (flags & PROCMAP_QUERY_VMA_FLAGS) {
654 u32 perm = 0;
655
656 if (flags & PROCMAP_QUERY_VMA_READABLE)
657 perm |= VM_READ;
658 if (flags & PROCMAP_QUERY_VMA_WRITABLE)
659 perm |= VM_WRITE;
660 if (flags & PROCMAP_QUERY_VMA_EXECUTABLE)
661 perm |= VM_EXEC;
662 if (flags & PROCMAP_QUERY_VMA_SHARED)
663 perm |= VM_MAYSHARE;
664
665 if ((vma->vm_flags & perm) != perm)
666 goto skip_vma;
667 }
668
669 /* found covering VMA or user is OK with the matching next VMA */
670 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr)
671 return vma;
672
673skip_vma:
674 /*
675 * If the user needs closest matching VMA, keep iterating.
676 */
677 addr = vma->vm_end;
678 if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA)
679 goto next_vma;
680
681no_vma:
682 return ERR_PTR(-ENOENT);
683}
684
685static int do_procmap_query(struct mm_struct *mm, void __user *uarg)
686{
687 struct proc_maps_locking_ctx lock_ctx = { .mm = mm };
688 struct procmap_query karg;
689 struct vm_area_struct *vma;
690 struct file *vm_file = NULL;
691 const char *name = NULL;
692 char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL;
693 __u64 usize;
694 int err;
695
696 if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize)))
697 return -EFAULT;
698 /* argument struct can never be that large, reject abuse */
699 if (usize > PAGE_SIZE)
700 return -E2BIG;
701 /* argument struct should have at least query_flags and query_addr fields */
702 if (usize < offsetofend(struct procmap_query, query_addr))
703 return -EINVAL;
704 err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize);
705 if (err)
706 return err;
707
708 /* reject unknown flags */
709 if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK)
710 return -EINVAL;
711 /* either both buffer address and size are set, or both should be zero */
712 if (!!karg.vma_name_size != !!karg.vma_name_addr)
713 return -EINVAL;
714 if (!!karg.build_id_size != !!karg.build_id_addr)
715 return -EINVAL;
716
717 if (!mm || !mmget_not_zero(mm))
718 return -ESRCH;
719
720 err = query_vma_setup(&lock_ctx);
721 if (err) {
722 mmput(mm);
723 return err;
724 }
725
726 vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags);
727 if (IS_ERR(vma)) {
728 err = PTR_ERR(vma);
729 vma = NULL;
730 goto out;
731 }
732
733 karg.vma_start = vma->vm_start;
734 karg.vma_end = vma->vm_end;
735
736 karg.vma_flags = 0;
737 if (vma->vm_flags & VM_READ)
738 karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE;
739 if (vma->vm_flags & VM_WRITE)
740 karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE;
741 if (vma->vm_flags & VM_EXEC)
742 karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE;
743 if (vma->vm_flags & VM_MAYSHARE)
744 karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED;
745
746 karg.vma_page_size = vma_kernel_pagesize(vma);
747
748 if (vma->vm_file) {
749 const struct inode *inode = file_user_inode(vma->vm_file);
750
751 karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT;
752 karg.dev_major = MAJOR(inode->i_sb->s_dev);
753 karg.dev_minor = MINOR(inode->i_sb->s_dev);
754 karg.inode = inode->i_ino;
755 } else {
756 karg.vma_offset = 0;
757 karg.dev_major = 0;
758 karg.dev_minor = 0;
759 karg.inode = 0;
760 }
761
762 if (karg.vma_name_size) {
763 size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size);
764 const struct path *path;
765 const char *name_fmt;
766 size_t name_sz = 0;
767
768 get_vma_name(vma, &path, &name, &name_fmt);
769
770 if (path || name_fmt || name) {
771 name_buf = kmalloc(name_buf_sz, GFP_KERNEL);
772 if (!name_buf) {
773 err = -ENOMEM;
774 goto out;
775 }
776 }
777 if (path) {
778 name = d_path(path, name_buf, name_buf_sz);
779 if (IS_ERR(name)) {
780 err = PTR_ERR(name);
781 goto out;
782 }
783 name_sz = name_buf + name_buf_sz - name;
784 } else if (name || name_fmt) {
785 name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name);
786 name = name_buf;
787 }
788 if (name_sz > name_buf_sz) {
789 err = -ENAMETOOLONG;
790 goto out;
791 }
792 karg.vma_name_size = name_sz;
793 }
794
795 if (karg.build_id_size && vma->vm_file)
796 vm_file = get_file(vma->vm_file);
797
798 /* unlock vma or mmap_lock, and put mm_struct before copying data to user */
799 query_vma_teardown(&lock_ctx);
800 mmput(mm);
801
802 if (karg.build_id_size) {
803 __u32 build_id_sz;
804
805 if (vm_file)
806 err = build_id_parse_file(vm_file, build_id_buf, &build_id_sz);
807 else
808 err = -ENOENT;
809 if (err) {
810 karg.build_id_size = 0;
811 } else {
812 if (karg.build_id_size < build_id_sz) {
813 err = -ENAMETOOLONG;
814 goto out_file;
815 }
816 karg.build_id_size = build_id_sz;
817 }
818 }
819
820 if (vm_file)
821 fput(vm_file);
822
823 if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr),
824 name, karg.vma_name_size)) {
825 kfree(name_buf);
826 return -EFAULT;
827 }
828 kfree(name_buf);
829
830 if (karg.build_id_size && copy_to_user(u64_to_user_ptr(karg.build_id_addr),
831 build_id_buf, karg.build_id_size))
832 return -EFAULT;
833
834 if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize)))
835 return -EFAULT;
836
837 return 0;
838
839out:
840 query_vma_teardown(&lock_ctx);
841 mmput(mm);
842out_file:
843 if (vm_file)
844 fput(vm_file);
845 kfree(name_buf);
846 return err;
847}
848
849static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
850{
851 struct seq_file *seq = file->private_data;
852 struct proc_maps_private *priv = seq->private;
853
854 switch (cmd) {
855 case PROCMAP_QUERY:
856 /* priv->lock_ctx.mm is set during file open operation */
857 return do_procmap_query(priv->lock_ctx.mm, (void __user *)arg);
858 default:
859 return -ENOIOCTLCMD;
860 }
861}
862
863const struct file_operations proc_pid_maps_operations = {
864 .open = pid_maps_open,
865 .read = seq_read,
866 .llseek = seq_lseek,
867 .release = proc_map_release,
868 .unlocked_ioctl = procfs_procmap_ioctl,
869 .compat_ioctl = compat_ptr_ioctl,
870};
871
872/*
873 * Proportional Set Size(PSS): my share of RSS.
874 *
875 * PSS of a process is the count of pages it has in memory, where each
876 * page is divided by the number of processes sharing it. So if a
877 * process has 1000 pages all to itself, and 1000 shared with one other
878 * process, its PSS will be 1500.
879 *
880 * To keep (accumulated) division errors low, we adopt a 64bit
881 * fixed-point pss counter to minimize division errors. So (pss >>
882 * PSS_SHIFT) would be the real byte count.
883 *
884 * A shift of 12 before division means (assuming 4K page size):
885 * - 1M 3-user-pages add up to 8KB errors;
886 * - supports mapcount up to 2^24, or 16M;
887 * - supports PSS up to 2^52 bytes, or 4PB.
888 */
889#define PSS_SHIFT 12
890
891#ifdef CONFIG_PROC_PAGE_MONITOR
892struct mem_size_stats {
893 unsigned long resident;
894 unsigned long shared_clean;
895 unsigned long shared_dirty;
896 unsigned long private_clean;
897 unsigned long private_dirty;
898 unsigned long referenced;
899 unsigned long anonymous;
900 unsigned long lazyfree;
901 unsigned long anonymous_thp;
902 unsigned long shmem_thp;
903 unsigned long file_thp;
904 unsigned long swap;
905 unsigned long shared_hugetlb;
906 unsigned long private_hugetlb;
907 unsigned long ksm;
908 u64 pss;
909 u64 pss_anon;
910 u64 pss_file;
911 u64 pss_shmem;
912 u64 pss_dirty;
913 u64 pss_locked;
914 u64 swap_pss;
915};
916
917static void smaps_page_accumulate(struct mem_size_stats *mss,
918 struct folio *folio, unsigned long size, unsigned long pss,
919 bool dirty, bool locked, bool private)
920{
921 mss->pss += pss;
922
923 if (folio_test_anon(folio))
924 mss->pss_anon += pss;
925 else if (folio_test_swapbacked(folio))
926 mss->pss_shmem += pss;
927 else
928 mss->pss_file += pss;
929
930 if (locked)
931 mss->pss_locked += pss;
932
933 if (dirty || folio_test_dirty(folio)) {
934 mss->pss_dirty += pss;
935 if (private)
936 mss->private_dirty += size;
937 else
938 mss->shared_dirty += size;
939 } else {
940 if (private)
941 mss->private_clean += size;
942 else
943 mss->shared_clean += size;
944 }
945}
946
947static void smaps_account(struct mem_size_stats *mss, struct page *page,
948 bool compound, bool young, bool dirty, bool locked,
949 bool present)
950{
951 struct folio *folio = page_folio(page);
952 int i, nr = compound ? compound_nr(page) : 1;
953 unsigned long size = nr * PAGE_SIZE;
954 bool exclusive;
955 int mapcount;
956
957 /*
958 * First accumulate quantities that depend only on |size| and the type
959 * of the compound page.
960 */
961 if (folio_test_anon(folio)) {
962 mss->anonymous += size;
963 if (!folio_test_swapbacked(folio) && !dirty &&
964 !folio_test_dirty(folio))
965 mss->lazyfree += size;
966 }
967
968 if (folio_test_ksm(folio))
969 mss->ksm += size;
970
971 mss->resident += size;
972 /* Accumulate the size in pages that have been accessed. */
973 if (young || folio_test_young(folio) || folio_test_referenced(folio))
974 mss->referenced += size;
975
976 /*
977 * Then accumulate quantities that may depend on sharing, or that may
978 * differ page-by-page.
979 *
980 * refcount == 1 for present entries guarantees that the folio is mapped
981 * exactly once. For large folios this implies that exactly one
982 * PTE/PMD/... maps (a part of) this folio.
983 *
984 * Treat all non-present entries (where relying on the mapcount and
985 * refcount doesn't make sense) as "maybe shared, but not sure how
986 * often". We treat device private entries as being fake-present.
987 *
988 * Note that it would not be safe to read the mapcount especially for
989 * pages referenced by migration entries, even with the PTL held.
990 */
991 if (folio_ref_count(folio) == 1 || !present) {
992 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT,
993 dirty, locked, present);
994 return;
995 }
996
997 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
998 mapcount = folio_average_page_mapcount(folio);
999 exclusive = !folio_maybe_mapped_shared(folio);
1000 }
1001
1002 /*
1003 * We obtain a snapshot of the mapcount. Without holding the folio lock
1004 * this snapshot can be slightly wrong as we cannot always read the
1005 * mapcount atomically.
1006 */
1007 for (i = 0; i < nr; i++, page++) {
1008 unsigned long pss = PAGE_SIZE << PSS_SHIFT;
1009
1010 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) {
1011 mapcount = folio_precise_page_mapcount(folio, page);
1012 exclusive = mapcount < 2;
1013 }
1014
1015 if (mapcount >= 2)
1016 pss /= mapcount;
1017 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss,
1018 dirty, locked, exclusive);
1019 }
1020}
1021
1022#ifdef CONFIG_SHMEM
1023static int smaps_pte_hole(unsigned long addr, unsigned long end,
1024 __always_unused int depth, struct mm_walk *walk)
1025{
1026 struct mem_size_stats *mss = walk->private;
1027 struct vm_area_struct *vma = walk->vma;
1028
1029 mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping,
1030 linear_page_index(vma, addr),
1031 linear_page_index(vma, end));
1032
1033 return 0;
1034}
1035#else
1036#define smaps_pte_hole NULL
1037#endif /* CONFIG_SHMEM */
1038
1039static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk)
1040{
1041#ifdef CONFIG_SHMEM
1042 if (walk->ops->pte_hole) {
1043 /* depth is not used */
1044 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk);
1045 }
1046#endif
1047}
1048
1049static void smaps_pte_entry(pte_t *pte, unsigned long addr,
1050 struct mm_walk *walk)
1051{
1052 struct mem_size_stats *mss = walk->private;
1053 struct vm_area_struct *vma = walk->vma;
1054 bool locked = !!(vma->vm_flags & VM_LOCKED);
1055 struct page *page = NULL;
1056 bool present = false, young = false, dirty = false;
1057 pte_t ptent = ptep_get(pte);
1058
1059 if (pte_present(ptent)) {
1060 page = vm_normal_page(vma, addr, ptent);
1061 young = pte_young(ptent);
1062 dirty = pte_dirty(ptent);
1063 present = true;
1064 } else if (pte_none(ptent)) {
1065 smaps_pte_hole_lookup(addr, walk);
1066 } else {
1067 const softleaf_t entry = softleaf_from_pte(ptent);
1068
1069 if (softleaf_is_swap(entry)) {
1070 int mapcount;
1071
1072 mss->swap += PAGE_SIZE;
1073 mapcount = swp_swapcount(entry);
1074 if (mapcount >= 2) {
1075 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
1076
1077 do_div(pss_delta, mapcount);
1078 mss->swap_pss += pss_delta;
1079 } else {
1080 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
1081 }
1082 } else if (softleaf_has_pfn(entry)) {
1083 if (softleaf_is_device_private(entry))
1084 present = true;
1085 page = softleaf_to_page(entry);
1086 }
1087 }
1088
1089 if (!page)
1090 return;
1091
1092 smaps_account(mss, page, false, young, dirty, locked, present);
1093}
1094
1095#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1096static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
1097 struct mm_walk *walk)
1098{
1099 struct mem_size_stats *mss = walk->private;
1100 struct vm_area_struct *vma = walk->vma;
1101 bool locked = !!(vma->vm_flags & VM_LOCKED);
1102 struct page *page = NULL;
1103 bool present = false;
1104 struct folio *folio;
1105
1106 if (pmd_none(*pmd))
1107 return;
1108 if (pmd_present(*pmd)) {
1109 page = vm_normal_page_pmd(vma, addr, *pmd);
1110 present = true;
1111 } else if (unlikely(thp_migration_supported())) {
1112 const softleaf_t entry = softleaf_from_pmd(*pmd);
1113
1114 if (softleaf_has_pfn(entry))
1115 page = softleaf_to_page(entry);
1116 }
1117 if (IS_ERR_OR_NULL(page))
1118 return;
1119 folio = page_folio(page);
1120 if (folio_test_anon(folio))
1121 mss->anonymous_thp += HPAGE_PMD_SIZE;
1122 else if (folio_test_swapbacked(folio))
1123 mss->shmem_thp += HPAGE_PMD_SIZE;
1124 else if (folio_is_zone_device(folio))
1125 /* pass */;
1126 else
1127 mss->file_thp += HPAGE_PMD_SIZE;
1128
1129 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd),
1130 locked, present);
1131}
1132#else
1133static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
1134 struct mm_walk *walk)
1135{
1136}
1137#endif
1138
1139static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
1140 struct mm_walk *walk)
1141{
1142 struct vm_area_struct *vma = walk->vma;
1143 pte_t *pte;
1144 spinlock_t *ptl;
1145
1146 ptl = pmd_trans_huge_lock(pmd, vma);
1147 if (ptl) {
1148 smaps_pmd_entry(pmd, addr, walk);
1149 spin_unlock(ptl);
1150 goto out;
1151 }
1152
1153 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1154 if (!pte) {
1155 walk->action = ACTION_AGAIN;
1156 return 0;
1157 }
1158 for (; addr != end; pte++, addr += PAGE_SIZE)
1159 smaps_pte_entry(pte, addr, walk);
1160 pte_unmap_unlock(pte - 1, ptl);
1161out:
1162 cond_resched();
1163 return 0;
1164}
1165
1166static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
1167{
1168 /*
1169 * Don't forget to update Documentation/ on changes.
1170 *
1171 * The length of the second argument of mnemonics[]
1172 * needs to be 3 instead of previously set 2
1173 * (i.e. from [BITS_PER_LONG][2] to [BITS_PER_LONG][3])
1174 * to avoid spurious
1175 * -Werror=unterminated-string-initialization warning
1176 * with GCC 15
1177 */
1178 static const char mnemonics[BITS_PER_LONG][3] = {
1179 /*
1180 * In case if we meet a flag we don't know about.
1181 */
1182 [0 ... (BITS_PER_LONG-1)] = "??",
1183
1184 [ilog2(VM_READ)] = "rd",
1185 [ilog2(VM_WRITE)] = "wr",
1186 [ilog2(VM_EXEC)] = "ex",
1187 [ilog2(VM_SHARED)] = "sh",
1188 [ilog2(VM_MAYREAD)] = "mr",
1189 [ilog2(VM_MAYWRITE)] = "mw",
1190 [ilog2(VM_MAYEXEC)] = "me",
1191 [ilog2(VM_MAYSHARE)] = "ms",
1192 [ilog2(VM_GROWSDOWN)] = "gd",
1193 [ilog2(VM_PFNMAP)] = "pf",
1194 [ilog2(VM_MAYBE_GUARD)] = "gu",
1195 [ilog2(VM_LOCKED)] = "lo",
1196 [ilog2(VM_IO)] = "io",
1197 [ilog2(VM_SEQ_READ)] = "sr",
1198 [ilog2(VM_RAND_READ)] = "rr",
1199 [ilog2(VM_DONTCOPY)] = "dc",
1200 [ilog2(VM_DONTEXPAND)] = "de",
1201 [ilog2(VM_LOCKONFAULT)] = "lf",
1202 [ilog2(VM_ACCOUNT)] = "ac",
1203 [ilog2(VM_NORESERVE)] = "nr",
1204 [ilog2(VM_HUGETLB)] = "ht",
1205 [ilog2(VM_SYNC)] = "sf",
1206 [ilog2(VM_ARCH_1)] = "ar",
1207 [ilog2(VM_WIPEONFORK)] = "wf",
1208 [ilog2(VM_DONTDUMP)] = "dd",
1209#ifdef CONFIG_ARM64_BTI
1210 [ilog2(VM_ARM64_BTI)] = "bt",
1211#endif
1212#ifdef CONFIG_MEM_SOFT_DIRTY
1213 [ilog2(VM_SOFTDIRTY)] = "sd",
1214#endif
1215 [ilog2(VM_MIXEDMAP)] = "mm",
1216 [ilog2(VM_HUGEPAGE)] = "hg",
1217 [ilog2(VM_NOHUGEPAGE)] = "nh",
1218 [ilog2(VM_MERGEABLE)] = "mg",
1219 [ilog2(VM_UFFD_MISSING)]= "um",
1220 [ilog2(VM_UFFD_WP)] = "uw",
1221#ifdef CONFIG_ARM64_MTE
1222 [ilog2(VM_MTE)] = "mt",
1223 [ilog2(VM_MTE_ALLOWED)] = "",
1224#endif
1225#ifdef CONFIG_ARCH_HAS_PKEYS
1226 /* These come out via ProtectionKey: */
1227 [ilog2(VM_PKEY_BIT0)] = "",
1228 [ilog2(VM_PKEY_BIT1)] = "",
1229 [ilog2(VM_PKEY_BIT2)] = "",
1230#if CONFIG_ARCH_PKEY_BITS > 3
1231 [ilog2(VM_PKEY_BIT3)] = "",
1232#endif
1233#if CONFIG_ARCH_PKEY_BITS > 4
1234 [ilog2(VM_PKEY_BIT4)] = "",
1235#endif
1236#endif /* CONFIG_ARCH_HAS_PKEYS */
1237#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1238 [ilog2(VM_UFFD_MINOR)] = "ui",
1239#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
1240#ifdef CONFIG_USERFAULTFD_RWP
1241 [ilog2(VM_UFFD_RWP)] = "ur",
1242#endif
1243#ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK
1244 [ilog2(VM_SHADOW_STACK)] = "ss",
1245#endif
1246#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
1247 [ilog2(VM_DROPPABLE)] = "dp",
1248#endif
1249#ifdef CONFIG_64BIT
1250 [ilog2(VM_SEALED)] = "sl",
1251#endif
1252 };
1253 size_t i;
1254
1255 seq_puts(m, "VmFlags: ");
1256 for (i = 0; i < BITS_PER_LONG; i++) {
1257 if (!mnemonics[i][0])
1258 continue;
1259 if (vma->vm_flags & (1UL << i))
1260 seq_printf(m, "%s ", mnemonics[i]);
1261 }
1262 seq_putc(m, '\n');
1263}
1264
1265#ifdef CONFIG_HUGETLB_PAGE
1266static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
1267 unsigned long addr, unsigned long end,
1268 struct mm_walk *walk)
1269{
1270 struct mem_size_stats *mss = walk->private;
1271 struct vm_area_struct *vma = walk->vma;
1272 struct folio *folio = NULL;
1273 bool present = false;
1274 spinlock_t *ptl;
1275 pte_t ptent;
1276
1277 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
1278 ptent = huge_ptep_get(walk->mm, addr, pte);
1279 if (pte_present(ptent)) {
1280 folio = page_folio(pte_page(ptent));
1281 present = true;
1282 } else {
1283 const softleaf_t entry = softleaf_from_pte(ptent);
1284
1285 if (softleaf_has_pfn(entry))
1286 folio = softleaf_to_folio(entry);
1287 }
1288
1289 if (folio) {
1290 /* We treat non-present entries as "maybe shared". */
1291 if (!present || folio_maybe_mapped_shared(folio) ||
1292 hugetlb_pmd_shared(pte))
1293 mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
1294 else
1295 mss->private_hugetlb += huge_page_size(hstate_vma(vma));
1296 }
1297 spin_unlock(ptl);
1298 return 0;
1299}
1300#else
1301#define smaps_hugetlb_range NULL
1302#endif /* HUGETLB_PAGE */
1303
1304static const struct mm_walk_ops smaps_walk_ops = {
1305 .pmd_entry = smaps_pte_range,
1306 .hugetlb_entry = smaps_hugetlb_range,
1307 .walk_lock = PGWALK_RDLOCK,
1308};
1309
1310static const struct mm_walk_ops smaps_shmem_walk_ops = {
1311 .pmd_entry = smaps_pte_range,
1312 .hugetlb_entry = smaps_hugetlb_range,
1313 .pte_hole = smaps_pte_hole,
1314 .walk_lock = PGWALK_RDLOCK,
1315};
1316
1317#ifdef CONFIG_PER_VMA_LOCK
1318
1319static const struct mm_walk_ops smaps_walk_vma_lock_ops = {
1320 .pmd_entry = smaps_pte_range,
1321 .hugetlb_entry = smaps_hugetlb_range,
1322 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY,
1323};
1324
1325static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = {
1326 .pmd_entry = smaps_pte_range,
1327 .hugetlb_entry = smaps_hugetlb_range,
1328 .pte_hole = smaps_pte_hole,
1329 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY,
1330};
1331
1332static inline const struct mm_walk_ops *
1333get_smaps_walk_ops(struct proc_maps_private *priv)
1334{
1335 if (priv->lock_ctx.mmap_locked)
1336 return &smaps_walk_ops;
1337 return &smaps_walk_vma_lock_ops;
1338}
1339
1340static inline const struct mm_walk_ops *
1341get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
1342{
1343 if (priv->lock_ctx.mmap_locked)
1344 return &smaps_shmem_walk_ops;
1345 return &smaps_shmem_walk_vma_lock_ops;
1346}
1347
1348#else /* CONFIG_PER_VMA_LOCK */
1349
1350static inline const struct mm_walk_ops *
1351get_smaps_walk_ops(struct proc_maps_private *priv)
1352{
1353 return &smaps_walk_ops;
1354}
1355
1356static inline const struct mm_walk_ops *
1357get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
1358{
1359 return &smaps_shmem_walk_ops;
1360}
1361
1362#endif /* CONFIG_PER_VMA_LOCK */
1363
1364/*
1365 * Gather mem stats from @vma with the indicated beginning
1366 * address @start, and keep them in @mss.
1367 *
1368 * Use vm_start of @vma as the beginning address if @start is 0.
1369 */
1370static void smap_gather_stats(struct proc_maps_private *priv,
1371 struct vm_area_struct *vma,
1372 struct mem_size_stats *mss, unsigned long start)
1373{
1374 const struct mm_walk_ops *ops = get_smaps_walk_ops(priv);
1375
1376 /* Invalid start */
1377 if (start >= vma->vm_end)
1378 return;
1379
1380 if (vma == get_gate_vma(priv->lock_ctx.mm))
1381 return;
1382
1383 /* Might sleep. Drop RCU read lock but keep the VMA locked. */
1384 drop_rcu(priv);
1385
1386 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
1387 /*
1388 * For shared or readonly shmem mappings we know that all
1389 * swapped out pages belong to the shmem object, and we can
1390 * obtain the swap value much more efficiently. For private
1391 * writable mappings, we might have COW pages that are
1392 * not affected by the parent swapped out pages of the shmem
1393 * object, so we have to distinguish them during the page walk.
1394 * Unless we know that the shmem object (or the part mapped by
1395 * our VMA) has no swapped out pages at all.
1396 */
1397 unsigned long shmem_swapped = shmem_swap_usage(vma);
1398
1399 if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
1400 !(vma->vm_flags & VM_WRITE))) {
1401 mss->swap += shmem_swapped;
1402 } else {
1403 ops = get_smaps_shmem_walk_ops(priv);
1404 }
1405 }
1406
1407 if (!start)
1408 walk_page_vma(vma, ops, mss);
1409 else
1410 walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
1411
1412 reacquire_rcu(priv);
1413}
1414
1415#define SEQ_PUT_DEC(str, val) \
1416 seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
1417
1418/* Show the contents common for smaps and smaps_rollup */
1419static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
1420 bool rollup_mode)
1421{
1422 SEQ_PUT_DEC("Rss: ", mss->resident);
1423 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
1424 SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT);
1425 if (rollup_mode) {
1426 /*
1427 * These are meaningful only for smaps_rollup, otherwise two of
1428 * them are zero, and the other one is the same as Pss.
1429 */
1430 SEQ_PUT_DEC(" kB\nPss_Anon: ",
1431 mss->pss_anon >> PSS_SHIFT);
1432 SEQ_PUT_DEC(" kB\nPss_File: ",
1433 mss->pss_file >> PSS_SHIFT);
1434 SEQ_PUT_DEC(" kB\nPss_Shmem: ",
1435 mss->pss_shmem >> PSS_SHIFT);
1436 }
1437 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
1438 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
1439 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
1440 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
1441 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
1442 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
1443 SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm);
1444 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
1445 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
1446 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
1447 SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
1448 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
1449 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
1450 mss->private_hugetlb >> 10, 7);
1451 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
1452 SEQ_PUT_DEC(" kB\nSwapPss: ",
1453 mss->swap_pss >> PSS_SHIFT);
1454 SEQ_PUT_DEC(" kB\nLocked: ",
1455 mss->pss_locked >> PSS_SHIFT);
1456 seq_puts(m, " kB\n");
1457}
1458
1459static int show_smap(struct seq_file *m, void *v)
1460{
1461 struct proc_maps_private *priv = m->private;
1462 struct vm_area_struct *vma = v;
1463 struct mem_size_stats mss = {};
1464
1465 smap_gather_stats(priv, vma, &mss, 0);
1466
1467 show_map_vma(m, vma);
1468
1469 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
1470 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
1471 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
1472 seq_puts(m, " kB\n");
1473
1474 __show_smap(m, &mss, false);
1475
1476 seq_printf(m, "THPeligible: %8u\n",
1477 !!thp_vma_allowable_orders(vma, vma->vm_flags, TVA_SMAPS,
1478 THP_ORDERS_ALL));
1479
1480 if (arch_pkeys_enabled())
1481 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
1482 show_smap_vma_flags(m, vma);
1483
1484 return 0;
1485}
1486
1487static int show_smaps_rollup(struct seq_file *m, void *v)
1488{
1489 struct proc_maps_private *priv = m->private;
1490 struct mem_size_stats mss = {};
1491 struct mm_struct *mm = priv->lock_ctx.mm;
1492 struct vm_area_struct *vma;
1493 unsigned long vma_start = 0, last_vma_end = 0;
1494 int ret = 0;
1495 VMA_ITERATOR(vmi, mm, 0);
1496
1497 priv->task = get_proc_task(priv->inode);
1498 if (!priv->task)
1499 return -ESRCH;
1500
1501 if (!mm || !mmget_not_zero(mm)) {
1502 ret = -ESRCH;
1503 goto out_put_task;
1504 }
1505
1506 ret = lock_ctx_mm(&priv->lock_ctx);
1507 if (ret)
1508 goto out_put_mm;
1509
1510 hold_task_mempolicy(priv);
1511 vma = vma_next(&vmi);
1512
1513 if (unlikely(!vma))
1514 goto empty_set;
1515
1516 vma_start = vma->vm_start;
1517 do {
1518 smap_gather_stats(priv, vma, &mss, 0);
1519 last_vma_end = vma->vm_end;
1520
1521 /*
1522 * Release mmap_lock temporarily if someone wants to
1523 * access it for write request.
1524 */
1525 if (mmap_lock_is_contended(mm)) {
1526 vma_iter_invalidate(&vmi);
1527 unlock_ctx_mm(&priv->lock_ctx);
1528 ret = lock_ctx_mm(&priv->lock_ctx);
1529 if (ret) {
1530 release_task_mempolicy(priv);
1531 goto out_put_mm;
1532 }
1533
1534 /*
1535 * After dropping the lock, there are four cases to
1536 * consider. See the following example for explanation.
1537 *
1538 * +------+------+-----------+
1539 * | VMA1 | VMA2 | VMA3 |
1540 * +------+------+-----------+
1541 * | | | |
1542 * 4k 8k 16k 400k
1543 *
1544 * Suppose we drop the lock after reading VMA2 due to
1545 * contention, then we get:
1546 *
1547 * last_vma_end = 16k
1548 *
1549 * 1) VMA2 is freed, but VMA3 exists:
1550 *
1551 * vma_next(vmi) will return VMA3.
1552 * In this case, just continue from VMA3.
1553 *
1554 * 2) VMA2 still exists:
1555 *
1556 * vma_next(vmi) will return VMA3.
1557 * In this case, just continue from VMA3.
1558 *
1559 * 3) No more VMAs can be found:
1560 *
1561 * vma_next(vmi) will return NULL.
1562 * No more things to do, just break.
1563 *
1564 * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
1565 *
1566 * vma_next(vmi) will return VMA' whose range
1567 * contains last_vma_end.
1568 * Iterate VMA' from last_vma_end.
1569 */
1570 vma = vma_next(&vmi);
1571 /* Case 3 above */
1572 if (!vma)
1573 break;
1574
1575 /* Case 1 and 2 above */
1576 if (vma->vm_start >= last_vma_end) {
1577 smap_gather_stats(priv, vma, &mss, 0);
1578 last_vma_end = vma->vm_end;
1579 continue;
1580 }
1581
1582 /* Case 4 above */
1583 if (vma->vm_end > last_vma_end) {
1584 smap_gather_stats(priv, vma, &mss, last_vma_end);
1585 last_vma_end = vma->vm_end;
1586 }
1587 }
1588 } for_each_vma(vmi, vma);
1589
1590empty_set:
1591 show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
1592 seq_pad(m, ' ');
1593 seq_puts(m, "[rollup]\n");
1594
1595 __show_smap(m, &mss, true);
1596
1597 release_task_mempolicy(priv);
1598 unlock_ctx_mm(&priv->lock_ctx);
1599
1600out_put_mm:
1601 mmput(mm);
1602out_put_task:
1603 put_task_struct(priv->task);
1604 priv->task = NULL;
1605
1606 return ret;
1607}
1608#undef SEQ_PUT_DEC
1609
1610static const struct seq_operations proc_pid_smaps_op = {
1611 .start = m_start,
1612 .next = m_next,
1613 .stop = m_stop,
1614 .show = show_smap
1615};
1616
1617static int pid_smaps_open(struct inode *inode, struct file *file)
1618{
1619 return do_maps_open(inode, file, &proc_pid_smaps_op);
1620}
1621
1622static int smaps_rollup_open(struct inode *inode, struct file *file)
1623{
1624 int ret;
1625 struct proc_maps_private *priv;
1626
1627 priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT);
1628 if (!priv)
1629 return -ENOMEM;
1630
1631 ret = single_open(file, show_smaps_rollup, priv);
1632 if (ret)
1633 goto out_free;
1634
1635 priv->inode = inode;
1636 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ);
1637 if (IS_ERR_OR_NULL(priv->lock_ctx.mm)) {
1638 ret = priv->lock_ctx.mm ? PTR_ERR(priv->lock_ctx.mm) : -ESRCH;
1639
1640 single_release(inode, file);
1641 goto out_free;
1642 }
1643
1644 return 0;
1645
1646out_free:
1647 kfree(priv);
1648 return ret;
1649}
1650
1651static int smaps_rollup_release(struct inode *inode, struct file *file)
1652{
1653 struct seq_file *seq = file->private_data;
1654 struct proc_maps_private *priv = seq->private;
1655
1656 if (priv->lock_ctx.mm)
1657 mmdrop(priv->lock_ctx.mm);
1658
1659 kfree(priv);
1660 return single_release(inode, file);
1661}
1662
1663const struct file_operations proc_pid_smaps_operations = {
1664 .open = pid_smaps_open,
1665 .read = seq_read,
1666 .llseek = seq_lseek,
1667 .release = proc_map_release,
1668};
1669
1670const struct file_operations proc_pid_smaps_rollup_operations = {
1671 .open = smaps_rollup_open,
1672 .read = seq_read,
1673 .llseek = seq_lseek,
1674 .release = smaps_rollup_release,
1675};
1676
1677enum clear_refs_types {
1678 CLEAR_REFS_ALL = 1,
1679 CLEAR_REFS_ANON,
1680 CLEAR_REFS_MAPPED,
1681 CLEAR_REFS_SOFT_DIRTY,
1682 CLEAR_REFS_MM_HIWATER_RSS,
1683 CLEAR_REFS_LAST,
1684};
1685
1686struct clear_refs_private {
1687 enum clear_refs_types type;
1688};
1689
1690static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1691{
1692 struct folio *folio;
1693
1694 if (!pte_write(pte))
1695 return false;
1696 if (!vma_is_cow_mapping(vma))
1697 return false;
1698 if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)))
1699 return false;
1700 folio = vm_normal_folio(vma, addr, pte);
1701 if (!folio)
1702 return false;
1703 return folio_maybe_dma_pinned(folio);
1704}
1705
1706static inline void clear_soft_dirty(struct vm_area_struct *vma,
1707 unsigned long addr, pte_t *pte)
1708{
1709 if (!pgtable_supports_soft_dirty())
1710 return;
1711 /*
1712 * The soft-dirty tracker uses #PF-s to catch writes
1713 * to pages, so write-protect the pte as well. See the
1714 * Documentation/admin-guide/mm/soft-dirty.rst for full description
1715 * of how soft-dirty works.
1716 */
1717 pte_t ptent = ptep_get(pte);
1718
1719 if (pte_none(ptent))
1720 return;
1721
1722 if (pte_present(ptent)) {
1723 pte_t old_pte;
1724
1725 if (pte_is_pinned(vma, addr, ptent))
1726 return;
1727 old_pte = ptep_modify_prot_start(vma, addr, pte);
1728 ptent = pte_wrprotect(old_pte);
1729 ptent = pte_clear_soft_dirty(ptent);
1730 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
1731 } else {
1732 ptent = pte_swp_clear_soft_dirty(ptent);
1733 set_pte_at(vma->vm_mm, addr, pte, ptent);
1734 }
1735}
1736
1737#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
1738static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1739 unsigned long addr, pmd_t *pmdp)
1740{
1741 pmd_t old, pmd = *pmdp;
1742
1743 if (!pgtable_supports_soft_dirty())
1744 return;
1745
1746 if (pmd_present(pmd)) {
1747 /* See comment in change_huge_pmd() */
1748 old = pmdp_invalidate(vma, addr, pmdp);
1749 if (pmd_dirty(old))
1750 pmd = pmd_mkdirty(pmd);
1751 if (pmd_young(old))
1752 pmd = pmd_mkyoung(pmd);
1753
1754 pmd = pmd_wrprotect(pmd);
1755 pmd = pmd_clear_soft_dirty(pmd);
1756
1757 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1758 } else if (pmd_is_migration_entry(pmd)) {
1759 pmd = pmd_swp_clear_soft_dirty(pmd);
1760 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1761 }
1762}
1763#else
1764static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1765 unsigned long addr, pmd_t *pmdp)
1766{
1767}
1768#endif
1769
1770static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1771 unsigned long end, struct mm_walk *walk)
1772{
1773 struct clear_refs_private *cp = walk->private;
1774 struct vm_area_struct *vma = walk->vma;
1775 pte_t *pte, ptent;
1776 spinlock_t *ptl;
1777 struct folio *folio;
1778
1779 ptl = pmd_trans_huge_lock(pmd, vma);
1780 if (ptl) {
1781 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1782 clear_soft_dirty_pmd(vma, addr, pmd);
1783 goto out;
1784 }
1785
1786 if (!pmd_present(*pmd))
1787 goto out;
1788
1789 folio = pmd_folio(*pmd);
1790
1791 /* Clear accessed and referenced bits. */
1792 pmdp_test_and_clear_young(vma, addr, pmd);
1793 folio_test_clear_young(folio);
1794 folio_clear_referenced(folio);
1795out:
1796 spin_unlock(ptl);
1797 return 0;
1798 }
1799
1800 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1801 if (!pte) {
1802 walk->action = ACTION_AGAIN;
1803 return 0;
1804 }
1805 for (; addr != end; pte++, addr += PAGE_SIZE) {
1806 ptent = ptep_get(pte);
1807
1808 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1809 clear_soft_dirty(vma, addr, pte);
1810 continue;
1811 }
1812
1813 if (!pte_present(ptent))
1814 continue;
1815
1816 folio = vm_normal_folio(vma, addr, ptent);
1817 if (!folio)
1818 continue;
1819
1820 /* Clear accessed and referenced bits. */
1821 ptep_test_and_clear_young(vma, addr, pte);
1822 folio_test_clear_young(folio);
1823 folio_clear_referenced(folio);
1824 }
1825 pte_unmap_unlock(pte - 1, ptl);
1826 cond_resched();
1827 return 0;
1828}
1829
1830static int clear_refs_test_walk(unsigned long start, unsigned long end,
1831 struct mm_walk *walk)
1832{
1833 struct clear_refs_private *cp = walk->private;
1834 struct vm_area_struct *vma = walk->vma;
1835
1836 if (vma->vm_flags & VM_PFNMAP)
1837 return 1;
1838
1839 /*
1840 * Writing 1 to /proc/pid/clear_refs affects all pages.
1841 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1842 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1843 * Writing 4 to /proc/pid/clear_refs affects all pages.
1844 */
1845 if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1846 return 1;
1847 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1848 return 1;
1849 return 0;
1850}
1851
1852static const struct mm_walk_ops clear_refs_walk_ops = {
1853 .pmd_entry = clear_refs_pte_range,
1854 .test_walk = clear_refs_test_walk,
1855 .walk_lock = PGWALK_WRLOCK,
1856};
1857
1858static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1859 size_t count, loff_t *ppos)
1860{
1861 struct task_struct *task;
1862 char buffer[PROC_NUMBUF] = {};
1863 struct mm_struct *mm;
1864 struct vm_area_struct *vma;
1865 enum clear_refs_types type;
1866 int itype;
1867 int rv;
1868
1869 if (count > sizeof(buffer) - 1)
1870 count = sizeof(buffer) - 1;
1871 if (copy_from_user(buffer, buf, count))
1872 return -EFAULT;
1873 rv = kstrtoint(strstrip(buffer), 10, &itype);
1874 if (rv < 0)
1875 return rv;
1876 type = (enum clear_refs_types)itype;
1877 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1878 return -EINVAL;
1879
1880 task = get_proc_task(file_inode(file));
1881 if (!task)
1882 return -ESRCH;
1883 mm = get_task_mm(task);
1884 if (mm) {
1885 VMA_ITERATOR(vmi, mm, 0);
1886 struct mmu_notifier_range range;
1887 struct clear_refs_private cp = {
1888 .type = type,
1889 };
1890
1891 if (mmap_write_lock_killable(mm)) {
1892 count = -EINTR;
1893 goto out_mm;
1894 }
1895 if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1896 /*
1897 * Writing 5 to /proc/pid/clear_refs resets the peak
1898 * resident set size to this mm's current rss value.
1899 */
1900 reset_mm_hiwater_rss(mm);
1901 goto out_unlock;
1902 }
1903
1904 if (type == CLEAR_REFS_SOFT_DIRTY) {
1905 for_each_vma(vmi, vma) {
1906 if (!(vma->vm_flags & VM_SOFTDIRTY))
1907 continue;
1908 vm_flags_clear(vma, VM_SOFTDIRTY);
1909 vma_set_page_prot(vma);
1910 }
1911
1912 inc_tlb_flush_pending(mm);
1913 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1914 0, mm, 0, -1UL);
1915 mmu_notifier_invalidate_range_start(&range);
1916 }
1917 walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp);
1918 if (type == CLEAR_REFS_SOFT_DIRTY) {
1919 mmu_notifier_invalidate_range_end(&range);
1920 flush_tlb_mm(mm);
1921 dec_tlb_flush_pending(mm);
1922 }
1923out_unlock:
1924 mmap_write_unlock(mm);
1925out_mm:
1926 mmput(mm);
1927 }
1928 put_task_struct(task);
1929
1930 return count;
1931}
1932
1933const struct file_operations proc_clear_refs_operations = {
1934 .write = clear_refs_write,
1935 .llseek = noop_llseek,
1936};
1937
1938typedef struct {
1939 u64 pme;
1940} pagemap_entry_t;
1941
1942struct pagemapread {
1943 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
1944 pagemap_entry_t *buffer;
1945 bool show_pfn;
1946};
1947
1948#define PAGEMAP_WALK_SIZE (PMD_SIZE)
1949#define PAGEMAP_WALK_MASK (PMD_MASK)
1950
1951#define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1952#define PM_PFRAME_BITS 55
1953#define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1954#define PM_SOFT_DIRTY BIT_ULL(55)
1955#define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1956#define PM_UFFD_WP BIT_ULL(57)
1957#define PM_GUARD_REGION BIT_ULL(58)
1958#define PM_FILE BIT_ULL(61)
1959#define PM_SWAP BIT_ULL(62)
1960#define PM_PRESENT BIT_ULL(63)
1961
1962#define PM_END_OF_BUFFER 1
1963
1964static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1965{
1966 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1967}
1968
1969static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm)
1970{
1971 pm->buffer[pm->pos++] = *pme;
1972 if (pm->pos >= pm->len)
1973 return PM_END_OF_BUFFER;
1974 return 0;
1975}
1976
1977static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page)
1978{
1979 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
1980 return folio_precise_page_mapcount(folio, page) == 1;
1981 return !folio_maybe_mapped_shared(folio);
1982}
1983
1984static int pagemap_pte_hole(unsigned long start, unsigned long end,
1985 __always_unused int depth, struct mm_walk *walk)
1986{
1987 struct pagemapread *pm = walk->private;
1988 unsigned long addr = start;
1989 int err = 0;
1990
1991 while (addr < end) {
1992 struct vm_area_struct *vma = find_vma(walk->mm, addr);
1993 pagemap_entry_t pme = make_pme(0, 0);
1994 /* End of address space hole, which we mark as non-present. */
1995 unsigned long hole_end;
1996
1997 if (vma)
1998 hole_end = min(end, vma->vm_start);
1999 else
2000 hole_end = end;
2001
2002 for (; addr < hole_end; addr += PAGE_SIZE) {
2003 err = add_to_pagemap(&pme, pm);
2004 if (err)
2005 goto out;
2006 }
2007
2008 if (!vma)
2009 break;
2010
2011 /* Addresses in the VMA. */
2012 if (vma->vm_flags & VM_SOFTDIRTY)
2013 pme = make_pme(0, PM_SOFT_DIRTY);
2014 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
2015 err = add_to_pagemap(&pme, pm);
2016 if (err)
2017 goto out;
2018 }
2019 }
2020out:
2021 return err;
2022}
2023
2024static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
2025 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
2026{
2027 u64 frame = 0, flags = 0;
2028 struct page *page = NULL;
2029 struct folio *folio;
2030
2031 if (pte_none(pte))
2032 goto out;
2033
2034 if (pte_present(pte)) {
2035 if (pm->show_pfn)
2036 frame = pte_pfn(pte);
2037 flags |= PM_PRESENT;
2038 page = vm_normal_page(vma, addr, pte);
2039 if (pte_soft_dirty(pte))
2040 flags |= PM_SOFT_DIRTY;
2041 if (pte_uffd(pte))
2042 flags |= PM_UFFD_WP;
2043 } else {
2044 softleaf_t entry;
2045
2046 if (pte_swp_soft_dirty(pte))
2047 flags |= PM_SOFT_DIRTY;
2048 if (pte_swp_uffd(pte))
2049 flags |= PM_UFFD_WP;
2050 entry = softleaf_from_pte(pte);
2051 if (pm->show_pfn) {
2052 pgoff_t offset;
2053
2054 /*
2055 * For PFN swap offsets, keeping the offset field
2056 * to be PFN only to be compatible with old smaps.
2057 */
2058 if (softleaf_has_pfn(entry))
2059 offset = softleaf_to_pfn(entry);
2060 else
2061 offset = swp_offset(entry);
2062 frame = swp_type(entry) |
2063 (offset << MAX_SWAPFILES_SHIFT);
2064 }
2065 flags |= PM_SWAP;
2066 if (softleaf_has_pfn(entry))
2067 page = softleaf_to_page(entry);
2068 if (softleaf_is_uffd_wp_marker(entry))
2069 flags |= PM_UFFD_WP;
2070 if (softleaf_is_guard_marker(entry))
2071 flags |= PM_GUARD_REGION;
2072 }
2073
2074 if (page) {
2075 folio = page_folio(page);
2076 if (!folio_test_anon(folio))
2077 flags |= PM_FILE;
2078 if ((flags & PM_PRESENT) &&
2079 __folio_page_mapped_exclusively(folio, page))
2080 flags |= PM_MMAP_EXCLUSIVE;
2081 }
2082
2083out:
2084 if (vma->vm_flags & VM_SOFTDIRTY)
2085 flags |= PM_SOFT_DIRTY;
2086
2087 return make_pme(frame, flags);
2088}
2089
2090#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2091static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr,
2092 unsigned long end, struct vm_area_struct *vma,
2093 struct pagemapread *pm)
2094{
2095 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT;
2096 u64 flags = 0, frame = 0;
2097 pmd_t pmd = *pmdp;
2098 struct page *page = NULL;
2099 struct folio *folio = NULL;
2100 int err = 0;
2101
2102 if (vma->vm_flags & VM_SOFTDIRTY)
2103 flags |= PM_SOFT_DIRTY;
2104
2105 if (pmd_none(pmd))
2106 goto populate_pagemap;
2107
2108 if (pmd_present(pmd)) {
2109 page = pmd_page(pmd);
2110
2111 flags |= PM_PRESENT;
2112 if (pmd_soft_dirty(pmd))
2113 flags |= PM_SOFT_DIRTY;
2114 if (pmd_uffd(pmd))
2115 flags |= PM_UFFD_WP;
2116 if (pm->show_pfn)
2117 frame = pmd_pfn(pmd) + idx;
2118 } else if (thp_migration_supported()) {
2119 const softleaf_t entry = softleaf_from_pmd(pmd);
2120 unsigned long offset;
2121
2122 if (pm->show_pfn) {
2123 if (softleaf_has_pfn(entry))
2124 offset = softleaf_to_pfn(entry) + idx;
2125 else
2126 offset = swp_offset(entry) + idx;
2127 frame = swp_type(entry) |
2128 (offset << MAX_SWAPFILES_SHIFT);
2129 }
2130 flags |= PM_SWAP;
2131 if (pmd_swp_soft_dirty(pmd))
2132 flags |= PM_SOFT_DIRTY;
2133 if (pmd_swp_uffd(pmd))
2134 flags |= PM_UFFD_WP;
2135 if (softleaf_has_pfn(entry))
2136 page = softleaf_to_page(entry);
2137 }
2138
2139 if (page) {
2140 folio = page_folio(page);
2141 if (!folio_test_anon(folio))
2142 flags |= PM_FILE;
2143 }
2144
2145populate_pagemap:
2146 for (; addr != end; addr += PAGE_SIZE, idx++) {
2147 u64 cur_flags = flags;
2148 pagemap_entry_t pme;
2149
2150 if (folio && (flags & PM_PRESENT) &&
2151 __folio_page_mapped_exclusively(folio, page))
2152 cur_flags |= PM_MMAP_EXCLUSIVE;
2153
2154 pme = make_pme(frame, cur_flags);
2155 err = add_to_pagemap(&pme, pm);
2156 if (err)
2157 break;
2158 if (pm->show_pfn) {
2159 if (flags & PM_PRESENT)
2160 frame++;
2161 else if (flags & PM_SWAP)
2162 frame += (1 << MAX_SWAPFILES_SHIFT);
2163 }
2164 }
2165 return err;
2166}
2167#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2168
2169static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
2170 struct mm_walk *walk)
2171{
2172 struct vm_area_struct *vma = walk->vma;
2173 struct pagemapread *pm = walk->private;
2174 spinlock_t *ptl;
2175 pte_t *pte, *orig_pte;
2176 int err = 0;
2177
2178#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2179 ptl = pmd_trans_huge_lock(pmdp, vma);
2180 if (ptl) {
2181 err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm);
2182 spin_unlock(ptl);
2183 return err;
2184 }
2185#endif
2186
2187 /*
2188 * We can assume that @vma always points to a valid one and @end never
2189 * goes beyond vma->vm_end.
2190 */
2191 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
2192 if (!pte) {
2193 walk->action = ACTION_AGAIN;
2194 return err;
2195 }
2196 for (; addr < end; pte++, addr += PAGE_SIZE) {
2197 pagemap_entry_t pme;
2198
2199 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte));
2200 err = add_to_pagemap(&pme, pm);
2201 if (err)
2202 break;
2203 }
2204 pte_unmap_unlock(orig_pte, ptl);
2205
2206 cond_resched();
2207
2208 return err;
2209}
2210
2211#ifdef CONFIG_HUGETLB_PAGE
2212/* This function walks within one hugetlb entry in the single call */
2213static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
2214 unsigned long addr, unsigned long end,
2215 struct mm_walk *walk)
2216{
2217 struct pagemapread *pm = walk->private;
2218 struct vm_area_struct *vma = walk->vma;
2219 u64 flags = 0, frame = 0;
2220 spinlock_t *ptl;
2221 int err = 0;
2222 pte_t pte;
2223
2224 if (vma->vm_flags & VM_SOFTDIRTY)
2225 flags |= PM_SOFT_DIRTY;
2226
2227 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, ptep);
2228 pte = huge_ptep_get(walk->mm, addr, ptep);
2229 if (pte_present(pte)) {
2230 struct folio *folio = page_folio(pte_page(pte));
2231
2232 if (!folio_test_anon(folio))
2233 flags |= PM_FILE;
2234
2235 if (!folio_maybe_mapped_shared(folio) &&
2236 !hugetlb_pmd_shared(ptep))
2237 flags |= PM_MMAP_EXCLUSIVE;
2238
2239 if (huge_pte_uffd(pte))
2240 flags |= PM_UFFD_WP;
2241
2242 flags |= PM_PRESENT;
2243 if (pm->show_pfn)
2244 frame = pte_pfn(pte) +
2245 ((addr & ~hmask) >> PAGE_SHIFT);
2246 } else if (pte_swp_uffd_any(pte)) {
2247 flags |= PM_UFFD_WP;
2248 }
2249
2250 for (; addr != end; addr += PAGE_SIZE) {
2251 pagemap_entry_t pme = make_pme(frame, flags);
2252
2253 err = add_to_pagemap(&pme, pm);
2254 if (err)
2255 break;
2256 if (pm->show_pfn && (flags & PM_PRESENT))
2257 frame++;
2258 }
2259
2260 spin_unlock(ptl);
2261 cond_resched();
2262
2263 return err;
2264}
2265#else
2266#define pagemap_hugetlb_range NULL
2267#endif /* HUGETLB_PAGE */
2268
2269static const struct mm_walk_ops pagemap_ops = {
2270 .pmd_entry = pagemap_pmd_range,
2271 .pte_hole = pagemap_pte_hole,
2272 .hugetlb_entry = pagemap_hugetlb_range,
2273 .walk_lock = PGWALK_RDLOCK,
2274};
2275
2276/*
2277 * /proc/pid/pagemap - an array mapping virtual pages to pfns
2278 *
2279 * For each page in the address space, this file contains one 64-bit entry
2280 * consisting of the following:
2281 *
2282 * Bits 0-54 page frame number (PFN) if present
2283 * Bits 0-4 swap type if swapped
2284 * Bits 5-54 swap offset if swapped
2285 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
2286 * Bit 56 page exclusively mapped
2287 * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP)
2288 * Bit 58 pte is a guard region
2289 * Bits 59-60 zero
2290 * Bit 61 page is file-page or shared-anon
2291 * Bit 62 page swapped
2292 * Bit 63 page present
2293 *
2294 * If the page is not present but in swap, then the PFN contains an
2295 * encoding of the swap file number and the page's offset into the
2296 * swap. Unmapped pages return a null PFN. This allows determining
2297 * precisely which pages are mapped (or in swap) and comparing mapped
2298 * pages between processes.
2299 *
2300 * Efficient users of this interface will use /proc/pid/maps to
2301 * determine which areas of memory are actually mapped and llseek to
2302 * skip over unmapped regions.
2303 */
2304static ssize_t pagemap_read(struct file *file, char __user *buf,
2305 size_t count, loff_t *ppos)
2306{
2307 struct mm_struct *mm = file->private_data;
2308 struct pagemapread pm;
2309 unsigned long src;
2310 unsigned long svpfn;
2311 unsigned long start_vaddr;
2312 unsigned long end_vaddr;
2313 int ret = 0, copied = 0;
2314
2315 if (!mm || !mmget_not_zero(mm))
2316 goto out;
2317
2318 ret = -EINVAL;
2319 /* file position must be aligned */
2320 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
2321 goto out_mm;
2322
2323 ret = 0;
2324 if (!count)
2325 goto out_mm;
2326
2327 /* do not disclose physical addresses: attack vector */
2328 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
2329
2330 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
2331 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
2332 ret = -ENOMEM;
2333 if (!pm.buffer)
2334 goto out_mm;
2335
2336 src = *ppos;
2337 svpfn = src / PM_ENTRY_BYTES;
2338 end_vaddr = mm->task_size;
2339
2340 /* watch out for wraparound */
2341 start_vaddr = end_vaddr;
2342 if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) {
2343 unsigned long end;
2344
2345 ret = mmap_read_lock_killable(mm);
2346 if (ret)
2347 goto out_free;
2348 start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT);
2349 mmap_read_unlock(mm);
2350
2351 end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT);
2352 if (end >= start_vaddr && end < mm->task_size)
2353 end_vaddr = end;
2354 }
2355
2356 /* Ensure the address is inside the task */
2357 if (start_vaddr > mm->task_size)
2358 start_vaddr = end_vaddr;
2359
2360 ret = 0;
2361 while (count && (start_vaddr < end_vaddr)) {
2362 int len;
2363 unsigned long end;
2364
2365 pm.pos = 0;
2366 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
2367 /* overflow ? */
2368 if (end < start_vaddr || end > end_vaddr)
2369 end = end_vaddr;
2370 ret = mmap_read_lock_killable(mm);
2371 if (ret)
2372 goto out_free;
2373 ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
2374 mmap_read_unlock(mm);
2375 start_vaddr = end;
2376
2377 len = min(count, PM_ENTRY_BYTES * pm.pos);
2378 if (copy_to_user(buf, pm.buffer, len)) {
2379 ret = -EFAULT;
2380 goto out_free;
2381 }
2382 copied += len;
2383 buf += len;
2384 count -= len;
2385 }
2386 *ppos += copied;
2387 if (!ret || ret == PM_END_OF_BUFFER)
2388 ret = copied;
2389
2390out_free:
2391 kfree(pm.buffer);
2392out_mm:
2393 mmput(mm);
2394out:
2395 return ret;
2396}
2397
2398static int pagemap_open(struct inode *inode, struct file *file)
2399{
2400 struct mm_struct *mm;
2401
2402 mm = proc_mem_open(inode, PTRACE_MODE_READ);
2403 if (IS_ERR_OR_NULL(mm))
2404 return mm ? PTR_ERR(mm) : -ESRCH;
2405 file->private_data = mm;
2406 return 0;
2407}
2408
2409static int pagemap_release(struct inode *inode, struct file *file)
2410{
2411 struct mm_struct *mm = file->private_data;
2412
2413 if (mm)
2414 mmdrop(mm);
2415 return 0;
2416}
2417
2418#define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \
2419 PAGE_IS_FILE | PAGE_IS_PRESENT | \
2420 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \
2421 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \
2422 PAGE_IS_GUARD | PAGE_IS_ACCESSED)
2423#define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC)
2424
2425struct pagemap_scan_private {
2426 struct pm_scan_arg arg;
2427 unsigned long masks_of_interest, cur_vma_category;
2428 struct page_region *vec_buf;
2429 unsigned long vec_buf_len, vec_buf_index, found_pages;
2430 struct page_region __user *vec_out;
2431};
2432
2433static unsigned long pagemap_page_category(struct pagemap_scan_private *p,
2434 struct vm_area_struct *vma,
2435 unsigned long addr, pte_t pte)
2436{
2437 unsigned long categories;
2438
2439 if (pte_none(pte)) {
2440 /*
2441 * An unpopulated pte carries no uffd bit, i.e. it is not
2442 * write-protected. The PAGE_IS_WRITTEN fast path in
2443 * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA;
2444 * gate the pte_none report here the same way so the two paths
2445 * still agree. RWP has no such fast path and an unpopulated
2446 * page is not part of the RWP working set, so it is reported as
2447 * neither.
2448 */
2449 if (userfaultfd_wp(vma))
2450 return PAGE_IS_WRITTEN;
2451 return 0;
2452 }
2453
2454 if (pte_present(pte)) {
2455 struct page *page;
2456
2457 categories = PAGE_IS_PRESENT;
2458
2459 if (!pte_uffd(pte)) {
2460 if (userfaultfd_wp(vma))
2461 categories |= PAGE_IS_WRITTEN;
2462 if (userfaultfd_rwp(vma))
2463 categories |= PAGE_IS_ACCESSED;
2464 }
2465
2466 if (p->masks_of_interest & PAGE_IS_FILE) {
2467 page = vm_normal_page(vma, addr, pte);
2468 if (page && !PageAnon(page))
2469 categories |= PAGE_IS_FILE;
2470 }
2471
2472 if (is_zero_pfn(pte_pfn(pte)))
2473 categories |= PAGE_IS_PFNZERO;
2474 if (pte_soft_dirty(pte))
2475 categories |= PAGE_IS_SOFT_DIRTY;
2476 } else {
2477 softleaf_t entry;
2478
2479 categories = PAGE_IS_SWAPPED;
2480
2481 if (!pte_swp_uffd_any(pte)) {
2482 if (userfaultfd_wp(vma))
2483 categories |= PAGE_IS_WRITTEN;
2484 if (userfaultfd_rwp(vma))
2485 categories |= PAGE_IS_ACCESSED;
2486 }
2487
2488 entry = softleaf_from_pte(pte);
2489 if (softleaf_is_guard_marker(entry))
2490 categories |= PAGE_IS_GUARD;
2491 else if ((p->masks_of_interest & PAGE_IS_FILE) &&
2492 softleaf_has_pfn(entry) &&
2493 !folio_test_anon(softleaf_to_folio(entry)))
2494 categories |= PAGE_IS_FILE;
2495
2496 if (pte_swp_soft_dirty(pte))
2497 categories |= PAGE_IS_SOFT_DIRTY;
2498 }
2499
2500 return categories;
2501}
2502
2503static void make_uffd_wp_pte(struct vm_area_struct *vma,
2504 unsigned long addr, pte_t *pte, pte_t ptent)
2505{
2506 if (pte_present(ptent)) {
2507 pte_t old_pte;
2508
2509 old_pte = ptep_modify_prot_start(vma, addr, pte);
2510 ptent = pte_mkuffd(old_pte);
2511 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
2512 } else if (pte_none(ptent)) {
2513 set_pte_at(vma->vm_mm, addr, pte,
2514 make_pte_marker(PTE_MARKER_UFFD_WP));
2515 } else {
2516 ptent = pte_swp_mkuffd(ptent);
2517 set_pte_at(vma->vm_mm, addr, pte, ptent);
2518 }
2519}
2520
2521#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2522static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
2523 struct vm_area_struct *vma,
2524 unsigned long addr, pmd_t pmd)
2525{
2526 unsigned long categories = PAGE_IS_HUGE;
2527
2528 if (pmd_none(pmd))
2529 return categories;
2530
2531 if (pmd_present(pmd)) {
2532 struct page *page;
2533
2534 categories |= PAGE_IS_PRESENT;
2535 if (!pmd_uffd(pmd)) {
2536 if (userfaultfd_wp(vma))
2537 categories |= PAGE_IS_WRITTEN;
2538 if (userfaultfd_rwp(vma))
2539 categories |= PAGE_IS_ACCESSED;
2540 }
2541
2542 if (p->masks_of_interest & PAGE_IS_FILE) {
2543 page = vm_normal_page_pmd(vma, addr, pmd);
2544 if (page && !PageAnon(page))
2545 categories |= PAGE_IS_FILE;
2546 }
2547
2548 if (is_huge_zero_pmd(pmd))
2549 categories |= PAGE_IS_PFNZERO;
2550 if (pmd_soft_dirty(pmd))
2551 categories |= PAGE_IS_SOFT_DIRTY;
2552 } else {
2553 categories |= PAGE_IS_SWAPPED;
2554 if (!pmd_swp_uffd(pmd)) {
2555 if (userfaultfd_wp(vma))
2556 categories |= PAGE_IS_WRITTEN;
2557 if (userfaultfd_rwp(vma))
2558 categories |= PAGE_IS_ACCESSED;
2559 }
2560 if (pmd_swp_soft_dirty(pmd))
2561 categories |= PAGE_IS_SOFT_DIRTY;
2562
2563 if (p->masks_of_interest & PAGE_IS_FILE) {
2564 const softleaf_t entry = softleaf_from_pmd(pmd);
2565
2566 if (softleaf_has_pfn(entry) &&
2567 !folio_test_anon(softleaf_to_folio(entry)))
2568 categories |= PAGE_IS_FILE;
2569 }
2570 }
2571
2572 return categories;
2573}
2574
2575static void make_uffd_wp_pmd(struct vm_area_struct *vma,
2576 unsigned long addr, pmd_t *pmdp)
2577{
2578 pmd_t old, pmd = *pmdp;
2579
2580 if (pmd_present(pmd)) {
2581 old = pmdp_invalidate_ad(vma, addr, pmdp);
2582 pmd = pmd_mkuffd(old);
2583 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
2584 } else if (pmd_is_migration_entry(pmd)) {
2585 pmd = pmd_swp_mkuffd(pmd);
2586 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
2587 }
2588}
2589#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2590
2591#ifdef CONFIG_HUGETLB_PAGE
2592static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma,
2593 pte_t pte)
2594{
2595 unsigned long categories = PAGE_IS_HUGE;
2596
2597 if (pte_none(pte))
2598 return categories;
2599
2600 /*
2601 * According to pagemap_hugetlb_range(), file-backed HugeTLB
2602 * page cannot be swapped. So PAGE_IS_FILE is not checked for
2603 * swapped pages.
2604 */
2605 if (pte_present(pte)) {
2606 categories |= PAGE_IS_PRESENT;
2607
2608 if (!huge_pte_uffd(pte)) {
2609 if (userfaultfd_wp(vma))
2610 categories |= PAGE_IS_WRITTEN;
2611 if (userfaultfd_rwp(vma))
2612 categories |= PAGE_IS_ACCESSED;
2613 }
2614 if (!PageAnon(pte_page(pte)))
2615 categories |= PAGE_IS_FILE;
2616 if (is_zero_pfn(pte_pfn(pte)))
2617 categories |= PAGE_IS_PFNZERO;
2618 if (pte_soft_dirty(pte))
2619 categories |= PAGE_IS_SOFT_DIRTY;
2620 } else {
2621 categories |= PAGE_IS_SWAPPED;
2622
2623 if (!pte_swp_uffd_any(pte)) {
2624 if (userfaultfd_wp(vma))
2625 categories |= PAGE_IS_WRITTEN;
2626 if (userfaultfd_rwp(vma))
2627 categories |= PAGE_IS_ACCESSED;
2628 }
2629 if (pte_swp_soft_dirty(pte))
2630 categories |= PAGE_IS_SOFT_DIRTY;
2631 }
2632
2633 return categories;
2634}
2635
2636static void make_uffd_wp_huge_pte(struct vm_area_struct *vma,
2637 unsigned long addr, pte_t *ptep,
2638 pte_t ptent)
2639{
2640 const unsigned long psize = huge_page_size(hstate_vma(vma));
2641 softleaf_t entry;
2642
2643 if (huge_pte_none(ptent)) {
2644 set_huge_pte_at(vma->vm_mm, addr, ptep,
2645 make_pte_marker(PTE_MARKER_UFFD_WP), psize);
2646 return;
2647 }
2648
2649 entry = softleaf_from_pte(ptent);
2650 if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry))
2651 return;
2652
2653 if (softleaf_is_migration(entry)) {
2654 set_huge_pte_at(vma->vm_mm, addr, ptep,
2655 pte_swp_mkuffd(ptent), psize);
2656 } else {
2657 pte_t old_pte, new_pte;
2658
2659 old_pte = huge_ptep_modify_prot_start(vma, addr, ptep);
2660 new_pte = huge_pte_mkuffd(old_pte);
2661 huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte);
2662 }
2663}
2664#endif /* CONFIG_HUGETLB_PAGE */
2665
2666#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
2667static void pagemap_scan_backout_range(struct pagemap_scan_private *p,
2668 unsigned long addr, unsigned long end)
2669{
2670 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index];
2671
2672 if (!p->vec_buf)
2673 return;
2674
2675 if (cur_buf->start != addr)
2676 cur_buf->end = addr;
2677 else
2678 cur_buf->start = cur_buf->end = 0;
2679
2680 p->found_pages -= (end - addr) / PAGE_SIZE;
2681}
2682#endif
2683
2684static bool pagemap_scan_is_interesting_page(unsigned long categories,
2685 const struct pagemap_scan_private *p)
2686{
2687 categories ^= p->arg.category_inverted;
2688 if ((categories & p->arg.category_mask) != p->arg.category_mask)
2689 return false;
2690 if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask))
2691 return false;
2692
2693 return true;
2694}
2695
2696static bool pagemap_scan_is_interesting_vma(unsigned long categories,
2697 const struct pagemap_scan_private *p)
2698{
2699 unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED;
2700
2701 categories ^= p->arg.category_inverted;
2702 if ((categories & required) != required)
2703 return false;
2704
2705 return true;
2706}
2707
2708static int pagemap_scan_test_walk(unsigned long start, unsigned long end,
2709 struct mm_walk *walk)
2710{
2711 struct pagemap_scan_private *p = walk->private;
2712 struct vm_area_struct *vma = walk->vma;
2713 unsigned long vma_category = 0;
2714 bool wp_allowed = userfaultfd_wp_async(vma) &&
2715 userfaultfd_wp_use_markers(vma);
2716
2717 if (!wp_allowed) {
2718 /* User requested explicit failure over wp-async capability */
2719 if (p->arg.flags & PM_SCAN_CHECK_WPASYNC)
2720 return -EPERM;
2721 /*
2722 * User requires wr-protect, and allows silently skipping
2723 * unsupported vmas.
2724 */
2725 if (p->arg.flags & PM_SCAN_WP_MATCHING)
2726 return 1;
2727 /*
2728 * Then the request doesn't involve wr-protects at all,
2729 * fall through to the rest checks, and allow vma walk.
2730 */
2731 }
2732
2733 if (vma->vm_flags & VM_PFNMAP)
2734 return 1;
2735
2736 if (wp_allowed)
2737 vma_category |= PAGE_IS_WPALLOWED;
2738
2739 if (vma->vm_flags & VM_SOFTDIRTY)
2740 vma_category |= PAGE_IS_SOFT_DIRTY;
2741
2742 if (!pagemap_scan_is_interesting_vma(vma_category, p))
2743 return 1;
2744
2745 p->cur_vma_category = vma_category;
2746
2747 return 0;
2748}
2749
2750static bool pagemap_scan_push_range(unsigned long categories,
2751 struct pagemap_scan_private *p,
2752 unsigned long addr, unsigned long end)
2753{
2754 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index];
2755
2756 /*
2757 * When there is no output buffer provided at all, the sentinel values
2758 * won't match here. There is no other way for `cur_buf->end` to be
2759 * non-zero other than it being non-empty.
2760 */
2761 if (addr == cur_buf->end && categories == cur_buf->categories) {
2762 cur_buf->end = end;
2763 return true;
2764 }
2765
2766 if (cur_buf->end) {
2767 if (p->vec_buf_index >= p->vec_buf_len - 1)
2768 return false;
2769
2770 cur_buf = &p->vec_buf[++p->vec_buf_index];
2771 }
2772
2773 cur_buf->start = addr;
2774 cur_buf->end = end;
2775 cur_buf->categories = categories;
2776
2777 return true;
2778}
2779
2780static int pagemap_scan_output(unsigned long categories,
2781 struct pagemap_scan_private *p,
2782 unsigned long addr, unsigned long *end)
2783{
2784 unsigned long n_pages, total_pages;
2785 int ret = 0;
2786
2787 if (!p->vec_buf)
2788 return 0;
2789
2790 categories &= p->arg.return_mask;
2791
2792 n_pages = (*end - addr) / PAGE_SIZE;
2793 if (check_add_overflow(p->found_pages, n_pages, &total_pages) ||
2794 total_pages > p->arg.max_pages) {
2795 size_t n_too_much = total_pages - p->arg.max_pages;
2796 *end -= n_too_much * PAGE_SIZE;
2797 n_pages -= n_too_much;
2798 ret = -ENOSPC;
2799 }
2800
2801 if (!pagemap_scan_push_range(categories, p, addr, *end)) {
2802 *end = addr;
2803 n_pages = 0;
2804 ret = -ENOSPC;
2805 }
2806
2807 p->found_pages += n_pages;
2808 if (ret)
2809 p->arg.walk_end = *end;
2810
2811 return ret;
2812}
2813
2814static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start,
2815 unsigned long end, struct mm_walk *walk)
2816{
2817#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2818 struct pagemap_scan_private *p = walk->private;
2819 struct vm_area_struct *vma = walk->vma;
2820 unsigned long categories;
2821 spinlock_t *ptl;
2822 int ret = 0;
2823
2824 ptl = pmd_trans_huge_lock(pmd, vma);
2825 if (!ptl)
2826 return -ENOENT;
2827
2828 categories = p->cur_vma_category |
2829 pagemap_thp_category(p, vma, start, *pmd);
2830
2831 if (!pagemap_scan_is_interesting_page(categories, p))
2832 goto out_unlock;
2833
2834 ret = pagemap_scan_output(categories, p, start, &end);
2835 if (start == end)
2836 goto out_unlock;
2837
2838 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2839 goto out_unlock;
2840 if (~categories & PAGE_IS_WRITTEN)
2841 goto out_unlock;
2842
2843 /*
2844 * Break huge page into small pages if the WP operation
2845 * needs to be performed on a portion of the huge page.
2846 */
2847 if (end != start + HPAGE_SIZE) {
2848 spin_unlock(ptl);
2849 split_huge_pmd(vma, pmd, start);
2850 pagemap_scan_backout_range(p, start, end);
2851 /* Report as if there was no THP */
2852 return -ENOENT;
2853 }
2854
2855 make_uffd_wp_pmd(vma, start, pmd);
2856 flush_tlb_range(vma, start, end);
2857out_unlock:
2858 spin_unlock(ptl);
2859 return ret;
2860#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
2861 return -ENOENT;
2862#endif
2863}
2864
2865static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
2866 unsigned long end, struct mm_walk *walk)
2867{
2868 struct pagemap_scan_private *p = walk->private;
2869 struct vm_area_struct *vma = walk->vma;
2870 unsigned long addr, flush_end = 0;
2871 pte_t *pte, *start_pte;
2872 spinlock_t *ptl;
2873 int ret;
2874
2875 ret = pagemap_scan_thp_entry(pmd, start, end, walk);
2876 if (ret != -ENOENT)
2877 return ret;
2878
2879 ret = 0;
2880 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
2881 if (!pte) {
2882 walk->action = ACTION_AGAIN;
2883 return 0;
2884 }
2885
2886 lazy_mmu_mode_enable();
2887
2888 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) {
2889 /* Fast path for performing exclusive WP */
2890 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
2891 pte_t ptent = ptep_get(pte);
2892
2893 if ((pte_present(ptent) && pte_uffd(ptent)) ||
2894 pte_swp_uffd_any(ptent))
2895 continue;
2896 make_uffd_wp_pte(vma, addr, pte, ptent);
2897 if (!flush_end)
2898 start = addr;
2899 flush_end = addr + PAGE_SIZE;
2900 }
2901 goto flush_and_return;
2902 }
2903
2904 if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask &&
2905 !p->arg.category_inverted &&
2906 p->arg.category_mask == PAGE_IS_WRITTEN &&
2907 p->arg.return_mask == PAGE_IS_WRITTEN) {
2908 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) {
2909 unsigned long next = addr + PAGE_SIZE;
2910 pte_t ptent = ptep_get(pte);
2911
2912 if ((pte_present(ptent) && pte_uffd(ptent)) ||
2913 pte_swp_uffd_any(ptent))
2914 continue;
2915 ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN,
2916 p, addr, &next);
2917 if (next == addr)
2918 break;
2919 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2920 continue;
2921 make_uffd_wp_pte(vma, addr, pte, ptent);
2922 if (!flush_end)
2923 start = addr;
2924 flush_end = next;
2925 }
2926 goto flush_and_return;
2927 }
2928
2929 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
2930 pte_t ptent = ptep_get(pte);
2931 unsigned long categories = p->cur_vma_category |
2932 pagemap_page_category(p, vma, addr, ptent);
2933 unsigned long next = addr + PAGE_SIZE;
2934
2935 if (!pagemap_scan_is_interesting_page(categories, p))
2936 continue;
2937
2938 ret = pagemap_scan_output(categories, p, addr, &next);
2939 if (next == addr)
2940 break;
2941
2942 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
2943 continue;
2944 if (~categories & PAGE_IS_WRITTEN)
2945 continue;
2946
2947 make_uffd_wp_pte(vma, addr, pte, ptent);
2948 if (!flush_end)
2949 start = addr;
2950 flush_end = next;
2951 }
2952
2953flush_and_return:
2954 if (flush_end)
2955 flush_tlb_range(vma, start, addr);
2956
2957 lazy_mmu_mode_disable();
2958 pte_unmap_unlock(start_pte, ptl);
2959
2960 cond_resched();
2961 return ret;
2962}
2963
2964#ifdef CONFIG_HUGETLB_PAGE
2965static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask,
2966 unsigned long start, unsigned long end,
2967 struct mm_walk *walk)
2968{
2969 struct pagemap_scan_private *p = walk->private;
2970 struct vm_area_struct *vma = walk->vma;
2971 unsigned long categories;
2972 spinlock_t *ptl;
2973 int ret = 0;
2974 pte_t pte;
2975
2976 if (~p->arg.flags & PM_SCAN_WP_MATCHING) {
2977 /* Go the short route when not write-protecting pages. */
2978
2979 pte = huge_ptep_get(walk->mm, start, ptep);
2980 categories = p->cur_vma_category |
2981 pagemap_hugetlb_category(vma, pte);
2982
2983 if (!pagemap_scan_is_interesting_page(categories, p))
2984 return 0;
2985
2986 return pagemap_scan_output(categories, p, start, &end);
2987 }
2988
2989 i_mmap_lock_write(vma->vm_file->f_mapping);
2990 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep);
2991
2992 pte = huge_ptep_get(walk->mm, start, ptep);
2993 categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte);
2994
2995 if (!pagemap_scan_is_interesting_page(categories, p))
2996 goto out_unlock;
2997
2998 ret = pagemap_scan_output(categories, p, start, &end);
2999 if (start == end)
3000 goto out_unlock;
3001
3002 if (~categories & PAGE_IS_WRITTEN)
3003 goto out_unlock;
3004
3005 if (end != start + huge_page_size(hstate_vma(vma))) {
3006 /* Partial HugeTLB page WP isn't possible. */
3007 pagemap_scan_backout_range(p, start, end);
3008 p->arg.walk_end = start;
3009 ret = 0;
3010 goto out_unlock;
3011 }
3012
3013 make_uffd_wp_huge_pte(vma, start, ptep, pte);
3014 flush_hugetlb_tlb_range(vma, start, end);
3015
3016out_unlock:
3017 spin_unlock(ptl);
3018 i_mmap_unlock_write(vma->vm_file->f_mapping);
3019
3020 return ret;
3021}
3022
3023/*
3024 * Write-protect the unpopulated hugetlb entries covering [addr, end) by
3025 * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry()
3026 * does for populated entries.
3027 *
3028 * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the
3029 * loop normally runs a single iteration; it is written to cover the full range
3030 * in case the walker ever coalesces adjacent holes.
3031 *
3032 * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() --
3033 * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table
3034 * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the
3035 * same VMA, so the scanning thread would deadlock against itself. PMD sharing
3036 * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so
3037 * the vma lock guards nothing that matters for these entries anyway.
3038 */
3039static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma,
3040 unsigned long addr, unsigned long end)
3041{
3042 struct hstate *h = hstate_vma(vma);
3043 unsigned long psize = huge_page_size(h);
3044 struct mm_struct *mm = vma->vm_mm;
3045 spinlock_t *ptl;
3046 pte_t *ptep;
3047 pte_t pte;
3048
3049 for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) {
3050 ptep = huge_pte_alloc(mm, vma, addr, psize);
3051 if (!ptep)
3052 return -ENOMEM;
3053
3054 i_mmap_lock_write(vma->vm_file->f_mapping);
3055 ptl = huge_pte_lock(h, mm, ptep);
3056 pte = huge_ptep_get(mm, addr, ptep);
3057 make_uffd_wp_huge_pte(vma, addr, ptep, pte);
3058 /*
3059 * A none entry has no cached translation, so installing the
3060 * marker needs no TLB flush. Flush only if a fault populated
3061 * the entry between huge_pte_alloc() and the page table lock.
3062 */
3063 if (!huge_pte_none(pte))
3064 flush_hugetlb_tlb_range(vma, addr, addr + psize);
3065 spin_unlock(ptl);
3066 i_mmap_unlock_write(vma->vm_file->f_mapping);
3067 }
3068
3069 return 0;
3070}
3071#else
3072#define pagemap_scan_hugetlb_entry NULL
3073static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma,
3074 unsigned long addr, unsigned long end)
3075{
3076 return 0;
3077}
3078#endif
3079
3080static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
3081 int depth, struct mm_walk *walk)
3082{
3083 struct pagemap_scan_private *p = walk->private;
3084 struct vm_area_struct *vma = walk->vma;
3085 unsigned long categories;
3086 int ret, err;
3087
3088 if (!vma)
3089 return 0;
3090
3091 /*
3092 * In a uffd-wp VMA an unpopulated range is treated as written:
3093 * uffd-wp registration populates page tables and installs markers
3094 * with WP_UNPOPULATED, so a missing marker means the range was
3095 * zapped. See the pte_none() handling in pagemap_page_category().
3096 *
3097 * hugetlb differs, see pagemap_hugetlb_category().
3098 */
3099 categories = p->cur_vma_category;
3100 if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma))
3101 categories |= PAGE_IS_WRITTEN;
3102
3103 if (!pagemap_scan_is_interesting_page(categories, p))
3104 return 0;
3105
3106 ret = pagemap_scan_output(categories, p, addr, &end);
3107 if (addr == end)
3108 return ret;
3109
3110 if (~p->arg.flags & PM_SCAN_WP_MATCHING)
3111 return ret;
3112
3113 if (is_vm_hugetlb_page(vma))
3114 err = pagemap_scan_hugetlb_hole_wp(vma, addr, end);
3115 else
3116 err = uffd_wp_range(vma, addr, end - addr, true);
3117 if (err < 0)
3118 ret = err;
3119
3120 return ret;
3121}
3122
3123static const struct mm_walk_ops pagemap_scan_ops = {
3124 .test_walk = pagemap_scan_test_walk,
3125 .pmd_entry = pagemap_scan_pmd_entry,
3126 .pte_hole = pagemap_scan_pte_hole,
3127 .hugetlb_entry = pagemap_scan_hugetlb_entry,
3128};
3129
3130static int pagemap_scan_get_args(struct pm_scan_arg *arg,
3131 unsigned long uarg)
3132{
3133 if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg)))
3134 return -EFAULT;
3135
3136 if (arg->size != sizeof(struct pm_scan_arg))
3137 return -EINVAL;
3138
3139 /* Validate requested features */
3140 if (arg->flags & ~PM_SCAN_FLAGS)
3141 return -EINVAL;
3142 if ((arg->category_inverted | arg->category_mask |
3143 arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES)
3144 return -EINVAL;
3145
3146 arg->start = untagged_addr((unsigned long)arg->start);
3147 arg->end = untagged_addr((unsigned long)arg->end);
3148 arg->vec = untagged_addr((unsigned long)arg->vec);
3149
3150 /* Validate memory pointers */
3151 if (!IS_ALIGNED(arg->start, PAGE_SIZE))
3152 return -EINVAL;
3153 if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start))
3154 return -EFAULT;
3155 if (!arg->vec && arg->vec_len)
3156 return -EINVAL;
3157 if (UINT_MAX == SIZE_MAX && arg->vec_len > SIZE_MAX)
3158 return -EINVAL;
3159 if (arg->vec && !access_ok((void __user *)(long)arg->vec,
3160 size_mul(arg->vec_len, sizeof(struct page_region))))
3161 return -EFAULT;
3162
3163 /* Fixup default values */
3164 arg->end = ALIGN(arg->end, PAGE_SIZE);
3165 arg->walk_end = 0;
3166 if (!arg->max_pages)
3167 arg->max_pages = ULONG_MAX;
3168
3169 return 0;
3170}
3171
3172static int pagemap_scan_writeback_args(struct pm_scan_arg *arg,
3173 unsigned long uargl)
3174{
3175 struct pm_scan_arg __user *uarg = (void __user *)uargl;
3176
3177 if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end)))
3178 return -EFAULT;
3179
3180 return 0;
3181}
3182
3183static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p)
3184{
3185 if (!p->arg.vec_len)
3186 return 0;
3187
3188 p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT,
3189 p->arg.vec_len);
3190 p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len);
3191 if (!p->vec_buf)
3192 return -ENOMEM;
3193
3194 p->vec_buf->start = p->vec_buf->end = 0;
3195 p->vec_out = (struct page_region __user *)(long)p->arg.vec;
3196
3197 return 0;
3198}
3199
3200static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p)
3201{
3202 const struct page_region *buf = p->vec_buf;
3203 long n = p->vec_buf_index;
3204
3205 if (!p->vec_buf)
3206 return 0;
3207
3208 if (buf[n].end != buf[n].start)
3209 n++;
3210
3211 if (!n)
3212 return 0;
3213
3214 if (copy_to_user(p->vec_out, buf, n * sizeof(*buf)))
3215 return -EFAULT;
3216
3217 p->arg.vec_len -= n;
3218 p->vec_out += n;
3219
3220 p->vec_buf_index = 0;
3221 p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len);
3222 p->vec_buf->start = p->vec_buf->end = 0;
3223
3224 return n;
3225}
3226
3227static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg)
3228{
3229 struct pagemap_scan_private p = {0};
3230 unsigned long walk_start;
3231 size_t n_ranges_out = 0;
3232 int ret;
3233
3234 ret = pagemap_scan_get_args(&p.arg, uarg);
3235 if (ret)
3236 return ret;
3237
3238 p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask |
3239 p.arg.return_mask;
3240 ret = pagemap_scan_init_bounce_buffer(&p);
3241 if (ret)
3242 return ret;
3243
3244 for (walk_start = p.arg.start; walk_start < p.arg.end;
3245 walk_start = p.arg.walk_end) {
3246 struct mmu_notifier_range range;
3247 long n_out;
3248
3249 if (fatal_signal_pending(current)) {
3250 ret = -EINTR;
3251 break;
3252 }
3253
3254 ret = mmap_read_lock_killable(mm);
3255 if (ret)
3256 break;
3257
3258 /* Protection change for the range is going to happen. */
3259 if (p.arg.flags & PM_SCAN_WP_MATCHING) {
3260 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0,
3261 mm, walk_start, p.arg.end);
3262 mmu_notifier_invalidate_range_start(&range);
3263 }
3264
3265 ret = walk_page_range(mm, walk_start, p.arg.end,
3266 &pagemap_scan_ops, &p);
3267
3268 if (p.arg.flags & PM_SCAN_WP_MATCHING)
3269 mmu_notifier_invalidate_range_end(&range);
3270
3271 mmap_read_unlock(mm);
3272
3273 n_out = pagemap_scan_flush_buffer(&p);
3274 if (n_out < 0)
3275 ret = n_out;
3276 else
3277 n_ranges_out += n_out;
3278
3279 if (ret != -ENOSPC)
3280 break;
3281
3282 if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages)
3283 break;
3284 }
3285
3286 /* ENOSPC signifies early stop (buffer full) from the walk. */
3287 if (!ret || ret == -ENOSPC)
3288 ret = n_ranges_out;
3289
3290 /* The walk_end isn't set when ret is zero */
3291 if (!p.arg.walk_end)
3292 p.arg.walk_end = p.arg.end;
3293 if (pagemap_scan_writeback_args(&p.arg, uarg))
3294 ret = -EFAULT;
3295
3296 kfree(p.vec_buf);
3297 return ret;
3298}
3299
3300static long do_pagemap_cmd(struct file *file, unsigned int cmd,
3301 unsigned long arg)
3302{
3303 struct mm_struct *mm = file->private_data;
3304
3305 switch (cmd) {
3306 case PAGEMAP_SCAN:
3307 return do_pagemap_scan(mm, arg);
3308
3309 default:
3310 return -EINVAL;
3311 }
3312}
3313
3314const struct file_operations proc_pagemap_operations = {
3315 .llseek = mem_lseek, /* borrow this */
3316 .read = pagemap_read,
3317 .open = pagemap_open,
3318 .release = pagemap_release,
3319 .unlocked_ioctl = do_pagemap_cmd,
3320 .compat_ioctl = do_pagemap_cmd,
3321};
3322#endif /* CONFIG_PROC_PAGE_MONITOR */
3323
3324#ifdef CONFIG_NUMA
3325
3326struct numa_maps {
3327 unsigned long pages;
3328 unsigned long anon;
3329 unsigned long active;
3330 unsigned long writeback;
3331 unsigned long mapcount_max;
3332 unsigned long dirty;
3333 unsigned long swapcache;
3334 unsigned long node[MAX_NUMNODES];
3335};
3336
3337struct numa_maps_private {
3338 struct proc_maps_private proc_maps;
3339 struct numa_maps md;
3340};
3341
3342static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
3343 unsigned long nr_pages)
3344{
3345 struct folio *folio = page_folio(page);
3346 int count;
3347
3348 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
3349 count = folio_precise_page_mapcount(folio, page);
3350 else
3351 count = folio_average_page_mapcount(folio);
3352
3353 md->pages += nr_pages;
3354 if (pte_dirty || folio_test_dirty(folio))
3355 md->dirty += nr_pages;
3356
3357 if (folio_test_swapcache(folio))
3358 md->swapcache += nr_pages;
3359
3360 if (folio_test_active(folio) || folio_test_unevictable(folio))
3361 md->active += nr_pages;
3362
3363 if (folio_test_writeback(folio))
3364 md->writeback += nr_pages;
3365
3366 if (folio_test_anon(folio))
3367 md->anon += nr_pages;
3368
3369 if (count > md->mapcount_max)
3370 md->mapcount_max = count;
3371
3372 md->node[folio_nid(folio)] += nr_pages;
3373}
3374
3375static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
3376 unsigned long addr)
3377{
3378 struct page *page;
3379 int nid;
3380
3381 if (!pte_present(pte))
3382 return NULL;
3383
3384 page = vm_normal_page(vma, addr, pte);
3385 if (!page || is_zone_device_page(page))
3386 return NULL;
3387
3388 if (PageReserved(page))
3389 return NULL;
3390
3391 nid = page_to_nid(page);
3392 if (!node_isset(nid, node_states[N_MEMORY]))
3393 return NULL;
3394
3395 return page;
3396}
3397
3398#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3399static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
3400 struct vm_area_struct *vma,
3401 unsigned long addr)
3402{
3403 struct page *page;
3404 int nid;
3405
3406 if (!pmd_present(pmd))
3407 return NULL;
3408
3409 page = vm_normal_page_pmd(vma, addr, pmd);
3410 if (!page)
3411 return NULL;
3412
3413 if (PageReserved(page))
3414 return NULL;
3415
3416 nid = page_to_nid(page);
3417 if (!node_isset(nid, node_states[N_MEMORY]))
3418 return NULL;
3419
3420 return page;
3421}
3422#endif
3423
3424static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
3425 unsigned long end, struct mm_walk *walk)
3426{
3427 struct numa_maps *md = walk->private;
3428 struct vm_area_struct *vma = walk->vma;
3429 spinlock_t *ptl;
3430 pte_t *orig_pte;
3431 pte_t *pte;
3432
3433#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3434 ptl = pmd_trans_huge_lock(pmd, vma);
3435 if (ptl) {
3436 struct page *page;
3437
3438 page = can_gather_numa_stats_pmd(*pmd, vma, addr);
3439 if (page)
3440 gather_stats(page, md, pmd_dirty(*pmd),
3441 HPAGE_PMD_SIZE/PAGE_SIZE);
3442 spin_unlock(ptl);
3443 return 0;
3444 }
3445#endif
3446 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
3447 if (!pte) {
3448 walk->action = ACTION_AGAIN;
3449 return 0;
3450 }
3451 do {
3452 pte_t ptent = ptep_get(pte);
3453 struct page *page = can_gather_numa_stats(ptent, vma, addr);
3454 if (!page)
3455 continue;
3456 gather_stats(page, md, pte_dirty(ptent), 1);
3457
3458 } while (pte++, addr += PAGE_SIZE, addr != end);
3459 pte_unmap_unlock(orig_pte, ptl);
3460 cond_resched();
3461 return 0;
3462}
3463#ifdef CONFIG_HUGETLB_PAGE
3464static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
3465 unsigned long addr, unsigned long end, struct mm_walk *walk)
3466{
3467 pte_t huge_pte;
3468 struct numa_maps *md;
3469 struct page *page;
3470 spinlock_t *ptl;
3471
3472 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
3473 huge_pte = huge_ptep_get(walk->mm, addr, pte);
3474 if (!pte_present(huge_pte))
3475 goto out;
3476
3477 page = pte_page(huge_pte);
3478
3479 md = walk->private;
3480 gather_stats(page, md, pte_dirty(huge_pte), 1);
3481out:
3482 spin_unlock(ptl);
3483 return 0;
3484}
3485
3486#else
3487static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
3488 unsigned long addr, unsigned long end, struct mm_walk *walk)
3489{
3490 return 0;
3491}
3492#endif
3493
3494static const struct mm_walk_ops show_numa_ops = {
3495 .hugetlb_entry = gather_hugetlb_stats,
3496 .pmd_entry = gather_pte_stats,
3497 .walk_lock = PGWALK_RDLOCK,
3498};
3499
3500#ifdef CONFIG_PER_VMA_LOCK
3501static const struct mm_walk_ops show_numa_vma_lock_ops = {
3502 .hugetlb_entry = gather_hugetlb_stats,
3503 .pmd_entry = gather_pte_stats,
3504 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY,
3505};
3506
3507static inline const struct mm_walk_ops *
3508get_show_numa_ops(struct proc_maps_private *priv)
3509{
3510 if (priv->lock_ctx.mmap_locked)
3511 return &show_numa_ops;
3512 return &show_numa_vma_lock_ops;
3513}
3514
3515#else /* CONFIG_PER_VMA_LOCK */
3516
3517static inline const struct mm_walk_ops *
3518get_show_numa_ops(struct proc_maps_private *priv)
3519{
3520 return &show_numa_ops;
3521}
3522
3523#endif /* CONFIG_PER_VMA_LOCK */
3524
3525/*
3526 * Display pages allocated per node and memory policy via /proc.
3527 */
3528static int show_numa_map(struct seq_file *m, void *v)
3529{
3530 struct numa_maps_private *numa_priv = m->private;
3531 struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
3532 struct vm_area_struct *vma = v;
3533 struct numa_maps *md = &numa_priv->md;
3534 struct file *file = vma->vm_file;
3535 struct mm_struct *mm = vma->vm_mm;
3536 char buffer[64];
3537 struct mempolicy *pol;
3538 pgoff_t ilx;
3539 int nid;
3540
3541 if (!mm)
3542 return 0;
3543
3544 /* Ensure we start with an empty set of numa_maps statistics. */
3545 memset(md, 0, sizeof(*md));
3546
3547 pol = __get_vma_policy(vma, vma->vm_start, &ilx);
3548 if (pol) {
3549 mpol_to_str(buffer, sizeof(buffer), pol);
3550 mpol_cond_put(pol);
3551 } else {
3552 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
3553 }
3554
3555 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
3556
3557 if (file) {
3558 seq_puts(m, " file=");
3559 seq_path(m, file_user_path(file), "\n\t= ");
3560 } else if (vma_is_initial_heap(vma)) {
3561 seq_puts(m, " heap");
3562 } else if (vma_is_initial_stack(vma)) {
3563 seq_puts(m, " stack");
3564 }
3565
3566 if (is_vm_hugetlb_page(vma))
3567 seq_puts(m, " huge");
3568
3569 /* Skip walking pages if gate VMA */
3570 if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) {
3571 /* Might sleep. Drop RCU read lock but keep the VMA locked. */
3572 drop_rcu(proc_priv);
3573 walk_page_vma(vma, get_show_numa_ops(proc_priv), md);
3574 reacquire_rcu(proc_priv);
3575 }
3576
3577 if (!md->pages)
3578 goto out;
3579
3580 if (md->anon)
3581 seq_printf(m, " anon=%lu", md->anon);
3582
3583 if (md->dirty)
3584 seq_printf(m, " dirty=%lu", md->dirty);
3585
3586 if (md->pages != md->anon && md->pages != md->dirty)
3587 seq_printf(m, " mapped=%lu", md->pages);
3588
3589 if (md->mapcount_max > 1)
3590 seq_printf(m, " mapmax=%lu", md->mapcount_max);
3591
3592 if (md->swapcache)
3593 seq_printf(m, " swapcache=%lu", md->swapcache);
3594
3595 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
3596 seq_printf(m, " active=%lu", md->active);
3597
3598 if (md->writeback)
3599 seq_printf(m, " writeback=%lu", md->writeback);
3600
3601 for_each_node_state(nid, N_MEMORY)
3602 if (md->node[nid])
3603 seq_printf(m, " N%d=%lu", nid, md->node[nid]);
3604
3605 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
3606out:
3607 seq_putc(m, '\n');
3608 return 0;
3609}
3610
3611static const struct seq_operations proc_pid_numa_maps_op = {
3612 .start = m_start,
3613 .next = m_next,
3614 .stop = m_stop,
3615 .show = show_numa_map,
3616};
3617
3618static int pid_numa_maps_open(struct inode *inode, struct file *file)
3619{
3620 return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
3621 sizeof(struct numa_maps_private));
3622}
3623
3624const struct file_operations proc_pid_numa_maps_operations = {
3625 .open = pid_numa_maps_open,
3626 .read = seq_read,
3627 .llseek = seq_lseek,
3628 .release = proc_map_release,
3629};
3630
3631#endif /* CONFIG_NUMA */