| 1 | |
| 2 | |
| 3 | * linux/mm/memory.c
|
| 4 | *
|
| 5 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
|
| 6 |
|
| 7 |
|
| 8 | |
| 9 | * demand-loading started 01.12.91 - seems it is high on the list of
|
| 10 | * things wanted, and it should be easy to implement. - Linus
|
| 11 |
|
| 12 |
|
| 13 | |
| 14 | * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
|
| 15 | * pages started 02.12.91, seems to work. - Linus.
|
| 16 | *
|
| 17 | * Tested sharing by executing about 30 /bin/sh: under the old kernel it
|
| 18 | * would have taken more than the 6M I have free, but it worked well as
|
| 19 | * far as I could see.
|
| 20 | *
|
| 21 | * Also corrected some "invalidate()"s - I wasn't doing enough of them.
|
| 22 |
|
| 23 |
|
| 24 | |
| 25 | * Real VM (paging to/from disk) started 18.12.91. Much more work and
|
| 26 | * thought has to go into this. Oh, well..
|
| 27 | * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
|
| 28 | * Found it. Everything seems to work now.
|
| 29 | * 20.12.91 - Ok, making the swap-device changeable like the root.
|
| 30 |
|
| 31 |
|
| 32 | |
| 33 | * 05.04.94 - Multi-page memory management added for v1.1.
|
| 34 | * Idea by Alex Bligh ([email protected])
|
| 35 | *
|
| 36 | * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
|
| 37 | * ([email protected])
|
| 38 | *
|
| 39 | * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
|
| 40 |
|
| 41 |
|
| 42 | #include <linux/kernel_stat.h>
|
| 43 | #include <linux/mm.h>
|
| 44 | #include <linux/mm_inline.h>
|
| 45 | #include <linux/sched/mm.h>
|
| 46 | #include <linux/sched/numa_balancing.h>
|
| 47 | #include <linux/sched/task.h>
|
| 48 | #include <linux/hugetlb.h>
|
| 49 | #include <linux/mman.h>
|
| 50 | #include <linux/swap.h>
|
| 51 | #include <linux/highmem.h>
|
| 52 | #include <linux/pagemap.h>
|
| 53 | #include <linux/memremap.h>
|
| 54 | #include <linux/kmsan.h>
|
| 55 | #include <linux/ksm.h>
|
| 56 | #include <linux/rmap.h>
|
| 57 | #include <linux/export.h>
|
| 58 | #include <linux/delayacct.h>
|
| 59 | #include <linux/init.h>
|
| 60 | #include <linux/writeback.h>
|
| 61 | #include <linux/memcontrol.h>
|
| 62 | #include <linux/mmu_notifier.h>
|
| 63 | #include <linux/leafops.h>
|
| 64 | #include <linux/elf.h>
|
| 65 | #include <linux/gfp.h>
|
| 66 | #include <linux/migrate.h>
|
| 67 | #include <linux/string.h>
|
| 68 | #include <linux/shmem_fs.h>
|
| 69 | #include <linux/memory-tiers.h>
|
| 70 | #include <linux/debugfs.h>
|
| 71 | #include <linux/userfaultfd_k.h>
|
| 72 | #include <linux/dax.h>
|
| 73 | #include <linux/oom.h>
|
| 74 | #include <linux/numa.h>
|
| 75 | #include <linux/perf_event.h>
|
| 76 | #include <linux/ptrace.h>
|
| 77 | #include <linux/vmalloc.h>
|
| 78 | #include <linux/sched/sysctl.h>
|
| 79 | #include <linux/pgalloc.h>
|
| 80 | #include <linux/uaccess.h>
|
| 81 |
|
| 82 | #include <trace/events/kmem.h>
|
| 83 |
|
| 84 | #include <asm/io.h>
|
| 85 | #include <asm/mmu_context.h>
|
| 86 | #include <asm/tlb.h>
|
| 87 | #include <asm/tlbflush.h>
|
| 88 |
|
| 89 | #include "pgalloc-track.h"
|
| 90 | #include "internal.h"
|
| 91 | #include "swap.h"
|
| 92 |
|
| 93 | #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
|
| 94 | #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
|
| 95 | #endif
|
| 96 |
|
| 97 | static vm_fault_t do_fault(struct vm_fault *vmf);
|
| 98 | static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
|
| 99 | static bool vmf_pte_changed(struct vm_fault *vmf);
|
| 100 |
|
| 101 | |
| 102 | * Return true if the original pte was a uffd-wp pte marker (so the pte was
|
| 103 | * wr-protected).
|
| 104 |
|
| 105 | static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
|
| 106 | {
|
| 107 | if (!userfaultfd_wp(vmf->vma))
|
| 108 | return false;
|
| 109 | if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
|
| 110 | return false;
|
| 111 |
|
| 112 | return pte_is_uffd_wp_marker(vmf->orig_pte);
|
| 113 | }
|
| 114 |
|
| 115 | |
| 116 | * Randomize the address space (stacks, mmaps, brk, etc.).
|
| 117 | *
|
| 118 | * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
|
| 119 | * as ancient (libc5 based) binaries can segfault. )
|
| 120 |
|
| 121 | int randomize_va_space __read_mostly =
|
| 122 | #ifdef CONFIG_COMPAT_BRK
|
| 123 | 1;
|
| 124 | #else
|
| 125 | 2;
|
| 126 | #endif
|
| 127 |
|
| 128 | static const struct ctl_table mmu_sysctl_table[] = {
|
| 129 | {
|
| 130 | .procname = "randomize_va_space",
|
| 131 | .data = &randomize_va_space,
|
| 132 | .maxlen = sizeof(int),
|
| 133 | .mode = 0644,
|
| 134 | .proc_handler = proc_dointvec,
|
| 135 | },
|
| 136 | };
|
| 137 |
|
| 138 | static int __init init_mm_sysctl(void)
|
| 139 | {
|
| 140 | register_sysctl_init("kernel", mmu_sysctl_table);
|
| 141 | return 0;
|
| 142 | }
|
| 143 |
|
| 144 | subsys_initcall(init_mm_sysctl);
|
| 145 |
|
| 146 | #ifndef arch_wants_old_prefaulted_pte
|
| 147 | static inline bool arch_wants_old_prefaulted_pte(void)
|
| 148 | {
|
| 149 | |
| 150 | * Transitioning a PTE from 'old' to 'young' can be expensive on
|
| 151 | * some architectures, even if it's performed in hardware. By
|
| 152 | * default, "false" means prefaulted entries will be 'young'.
|
| 153 |
|
| 154 | return false;
|
| 155 | }
|
| 156 | #endif
|
| 157 |
|
| 158 | static int __init disable_randmaps(char *s)
|
| 159 | {
|
| 160 | randomize_va_space = 0;
|
| 161 | return 1;
|
| 162 | }
|
| 163 | __setup("norandmaps", disable_randmaps);
|
| 164 |
|
| 165 | unsigned long highest_memmap_pfn __read_mostly;
|
| 166 |
|
| 167 | void mm_trace_rss_stat(struct mm_struct *mm, int member)
|
| 168 | {
|
| 169 | trace_rss_stat(mm, member);
|
| 170 | }
|
| 171 |
|
| 172 | |
| 173 | * Note: this doesn't free the actual pages themselves. That
|
| 174 | * has been handled earlier when unmapping all the memory regions.
|
| 175 |
|
| 176 | static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
|
| 177 | unsigned long addr)
|
| 178 | {
|
| 179 | pgtable_t token = pmd_pgtable(*pmd);
|
| 180 | pmd_clear(pmd);
|
| 181 | pte_free_tlb(tlb, token, addr);
|
| 182 | mm_dec_nr_ptes(tlb->mm);
|
| 183 | }
|
| 184 |
|
| 185 | static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
|
| 186 | unsigned long addr, unsigned long end,
|
| 187 | unsigned long floor, unsigned long ceiling)
|
| 188 | {
|
| 189 | pmd_t *pmd;
|
| 190 | unsigned long next;
|
| 191 | unsigned long start;
|
| 192 |
|
| 193 | start = addr;
|
| 194 | pmd = pmd_offset(pud, addr);
|
| 195 | do {
|
| 196 | next = pmd_addr_end(addr, end);
|
| 197 | if (pmd_none_or_clear_bad(pmd))
|
| 198 | continue;
|
| 199 | free_pte_range(tlb, pmd, addr);
|
| 200 | } while (pmd++, addr = next, addr != end);
|
| 201 |
|
| 202 | start &= PUD_MASK;
|
| 203 | if (start < floor)
|
| 204 | return;
|
| 205 | if (ceiling) {
|
| 206 | ceiling &= PUD_MASK;
|
| 207 | if (!ceiling)
|
| 208 | return;
|
| 209 | }
|
| 210 | if (end - 1 > ceiling - 1)
|
| 211 | return;
|
| 212 |
|
| 213 | pmd = pmd_offset(pud, start);
|
| 214 | pud_clear(pud);
|
| 215 | pmd_free_tlb(tlb, pmd, start);
|
| 216 | mm_dec_nr_pmds(tlb->mm);
|
| 217 | }
|
| 218 |
|
| 219 | static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
|
| 220 | unsigned long addr, unsigned long end,
|
| 221 | unsigned long floor, unsigned long ceiling)
|
| 222 | {
|
| 223 | pud_t *pud;
|
| 224 | unsigned long next;
|
| 225 | unsigned long start;
|
| 226 |
|
| 227 | start = addr;
|
| 228 | pud = pud_offset(p4d, addr);
|
| 229 | do {
|
| 230 | next = pud_addr_end(addr, end);
|
| 231 | if (pud_none_or_clear_bad(pud))
|
| 232 | continue;
|
| 233 | free_pmd_range(tlb, pud, addr, next, floor, ceiling);
|
| 234 | } while (pud++, addr = next, addr != end);
|
| 235 |
|
| 236 | start &= P4D_MASK;
|
| 237 | if (start < floor)
|
| 238 | return;
|
| 239 | if (ceiling) {
|
| 240 | ceiling &= P4D_MASK;
|
| 241 | if (!ceiling)
|
| 242 | return;
|
| 243 | }
|
| 244 | if (end - 1 > ceiling - 1)
|
| 245 | return;
|
| 246 |
|
| 247 | pud = pud_offset(p4d, start);
|
| 248 | p4d_clear(p4d);
|
| 249 | pud_free_tlb(tlb, pud, start);
|
| 250 | mm_dec_nr_puds(tlb->mm);
|
| 251 | }
|
| 252 |
|
| 253 | static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
|
| 254 | unsigned long addr, unsigned long end,
|
| 255 | unsigned long floor, unsigned long ceiling)
|
| 256 | {
|
| 257 | p4d_t *p4d;
|
| 258 | unsigned long next;
|
| 259 | unsigned long start;
|
| 260 |
|
| 261 | start = addr;
|
| 262 | p4d = p4d_offset(pgd, addr);
|
| 263 | do {
|
| 264 | next = p4d_addr_end(addr, end);
|
| 265 | if (p4d_none_or_clear_bad(p4d))
|
| 266 | continue;
|
| 267 | free_pud_range(tlb, p4d, addr, next, floor, ceiling);
|
| 268 | } while (p4d++, addr = next, addr != end);
|
| 269 |
|
| 270 | start &= PGDIR_MASK;
|
| 271 | if (start < floor)
|
| 272 | return;
|
| 273 | if (ceiling) {
|
| 274 | ceiling &= PGDIR_MASK;
|
| 275 | if (!ceiling)
|
| 276 | return;
|
| 277 | }
|
| 278 | if (end - 1 > ceiling - 1)
|
| 279 | return;
|
| 280 |
|
| 281 | p4d = p4d_offset(pgd, start);
|
| 282 | pgd_clear(pgd);
|
| 283 | p4d_free_tlb(tlb, p4d, start);
|
| 284 | }
|
| 285 |
|
| 286 | |
| 287 | * free_pgd_range - Unmap and free page tables in the range
|
| 288 | * @tlb: the mmu_gather containing pending TLB flush info
|
| 289 | * @addr: virtual address start
|
| 290 | * @end: virtual address end
|
| 291 | * @floor: lowest address boundary
|
| 292 | * @ceiling: highest address boundary
|
| 293 | *
|
| 294 | * This function tears down all user-level page tables in the
|
| 295 | * specified virtual address range [@addr..@end). It is part of
|
| 296 | * the memory unmap flow.
|
| 297 |
|
| 298 | void free_pgd_range(struct mmu_gather *tlb,
|
| 299 | unsigned long addr, unsigned long end,
|
| 300 | unsigned long floor, unsigned long ceiling)
|
| 301 | {
|
| 302 | pgd_t *pgd;
|
| 303 | unsigned long next;
|
| 304 |
|
| 305 | |
| 306 | * The next few lines have given us lots of grief...
|
| 307 | *
|
| 308 | * Why are we testing PMD* at this top level? Because often
|
| 309 | * there will be no work to do at all, and we'd prefer not to
|
| 310 | * go all the way down to the bottom just to discover that.
|
| 311 | *
|
| 312 | * Why all these "- 1"s? Because 0 represents both the bottom
|
| 313 | * of the address space and the top of it (using -1 for the
|
| 314 | * top wouldn't help much: the masks would do the wrong thing).
|
| 315 | * The rule is that addr 0 and floor 0 refer to the bottom of
|
| 316 | * the address space, but end 0 and ceiling 0 refer to the top
|
| 317 | * Comparisons need to use "end - 1" and "ceiling - 1" (though
|
| 318 | * that end 0 case should be mythical).
|
| 319 | *
|
| 320 | * Wherever addr is brought up or ceiling brought down, we must
|
| 321 | * be careful to reject "the opposite 0" before it confuses the
|
| 322 | * subsequent tests. But what about where end is brought down
|
| 323 | * by PMD_SIZE below? no, end can't go down to 0 there.
|
| 324 | *
|
| 325 | * Whereas we round start (addr) and ceiling down, by different
|
| 326 | * masks at different levels, in order to test whether a table
|
| 327 | * now has no other vmas using it, so can be freed, we don't
|
| 328 | * bother to round floor or end up - the tests don't need that.
|
| 329 |
|
| 330 |
|
| 331 | addr &= PMD_MASK;
|
| 332 | if (addr < floor) {
|
| 333 | addr += PMD_SIZE;
|
| 334 | if (!addr)
|
| 335 | return;
|
| 336 | }
|
| 337 | if (ceiling) {
|
| 338 | ceiling &= PMD_MASK;
|
| 339 | if (!ceiling)
|
| 340 | return;
|
| 341 | }
|
| 342 | if (end - 1 > ceiling - 1)
|
| 343 | end -= PMD_SIZE;
|
| 344 | if (addr > end - 1)
|
| 345 | return;
|
| 346 | |
| 347 | * We add page table cache pages with PAGE_SIZE,
|
| 348 | * (see pte_free_tlb()), flush the tlb if we need
|
| 349 |
|
| 350 | tlb_change_page_size(tlb, PAGE_SIZE);
|
| 351 | pgd = pgd_offset(tlb->mm, addr);
|
| 352 | do {
|
| 353 | next = pgd_addr_end(addr, end);
|
| 354 | if (pgd_none_or_clear_bad(pgd))
|
| 355 | continue;
|
| 356 | free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
|
| 357 | } while (pgd++, addr = next, addr != end);
|
| 358 | }
|
| 359 |
|
| 360 | |
| 361 | * free_pgtables() - Free a range of page tables
|
| 362 | * @tlb: The mmu gather
|
| 363 | * @unmap: The unmap_desc
|
| 364 | *
|
| 365 | * Note: pg_start and pg_end are provided to indicate the absolute range of the
|
| 366 | * page tables that should be removed. This can differ from the vma mappings on
|
| 367 | * some archs that may have mappings that need to be removed outside the vmas.
|
| 368 | * Note that the prev->vm_end and next->vm_start are often used.
|
| 369 | *
|
| 370 | * The vma_end differs from the pg_end when a dup_mmap() failed and the tree has
|
| 371 | * unrelated data to the mm_struct being torn down.
|
| 372 |
|
| 373 | void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *unmap)
|
| 374 | {
|
| 375 | struct unlink_vma_file_batch vb;
|
| 376 | struct ma_state *mas = unmap->mas;
|
| 377 | struct vm_area_struct *vma = unmap->first;
|
| 378 |
|
| 379 | |
| 380 | * Note: USER_PGTABLES_CEILING may be passed as the value of pg_end and
|
| 381 | * may be 0. Underflow is expected in this case. Otherwise the
|
| 382 | * pagetable end is exclusive. vma_end is exclusive. The last vma
|
| 383 | * address should never be larger than the pagetable end.
|
| 384 |
|
| 385 | WARN_ON_ONCE(unmap->vma_end - 1 > unmap->pg_end - 1);
|
| 386 |
|
| 387 | tlb_free_vmas(tlb);
|
| 388 |
|
| 389 | do {
|
| 390 | unsigned long addr = vma->vm_start;
|
| 391 | struct vm_area_struct *next;
|
| 392 |
|
| 393 | next = mas_find(mas, unmap->tree_end - 1);
|
| 394 |
|
| 395 | |
| 396 | * Hide vma from rmap and truncate_pagecache before freeing
|
| 397 | * pgtables
|
| 398 |
|
| 399 | if (unmap->mm_wr_locked)
|
| 400 | vma_start_write(vma);
|
| 401 | unlink_anon_vmas(vma);
|
| 402 |
|
| 403 | unlink_file_vma_batch_init(&vb);
|
| 404 | unlink_file_vma_batch_add(&vb, vma);
|
| 405 |
|
| 406 | |
| 407 | * Optimization: gather nearby vmas into one call down
|
| 408 |
|
| 409 | while (next && next->vm_start <= vma->vm_end + PMD_SIZE) {
|
| 410 | vma = next;
|
| 411 | next = mas_find(mas, unmap->tree_end - 1);
|
| 412 | if (unmap->mm_wr_locked)
|
| 413 | vma_start_write(vma);
|
| 414 | unlink_anon_vmas(vma);
|
| 415 | unlink_file_vma_batch_add(&vb, vma);
|
| 416 | }
|
| 417 | unlink_file_vma_batch_final(&vb);
|
| 418 |
|
| 419 | free_pgd_range(tlb, addr, vma->vm_end, unmap->pg_start,
|
| 420 | next ? next->vm_start : unmap->pg_end);
|
| 421 | vma = next;
|
| 422 | } while (vma);
|
| 423 | }
|
| 424 |
|
| 425 | void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
|
| 426 | {
|
| 427 | spinlock_t *ptl = pmd_lock(mm, pmd);
|
| 428 |
|
| 429 | if (likely(pmd_none(*pmd))) {
|
| 430 | mm_inc_nr_ptes(mm);
|
| 431 | |
| 432 | * Ensure all pte setup (eg. pte page lock and page clearing) are
|
| 433 | * visible before the pte is made visible to other CPUs by being
|
| 434 | * put into page tables.
|
| 435 | *
|
| 436 | * The other side of the story is the pointer chasing in the page
|
| 437 | * table walking code (when walking the page table without locking;
|
| 438 | * ie. most of the time). Fortunately, these data accesses consist
|
| 439 | * of a chain of data-dependent loads, meaning most CPUs (alpha
|
| 440 | * being the notable exception) will already guarantee loads are
|
| 441 | * seen in-order. See the alpha page table accessors for the
|
| 442 | * smp_rmb() barriers in page table walking code.
|
| 443 |
|
| 444 | smp_wmb();
|
| 445 | pmd_populate(mm, pmd, *pte);
|
| 446 | *pte = NULL;
|
| 447 | }
|
| 448 | spin_unlock(ptl);
|
| 449 | }
|
| 450 |
|
| 451 | int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
|
| 452 | {
|
| 453 | pgtable_t new = pte_alloc_one(mm);
|
| 454 | if (!new)
|
| 455 | return -ENOMEM;
|
| 456 |
|
| 457 | pmd_install(mm, pmd, &new);
|
| 458 | if (new)
|
| 459 | pte_free(mm, new);
|
| 460 | return 0;
|
| 461 | }
|
| 462 |
|
| 463 | int __pte_alloc_kernel(pmd_t *pmd)
|
| 464 | {
|
| 465 | pte_t *new = pte_alloc_one_kernel(&init_mm);
|
| 466 | if (!new)
|
| 467 | return -ENOMEM;
|
| 468 |
|
| 469 | spin_lock(&init_mm.page_table_lock);
|
| 470 | if (likely(pmd_none(*pmd))) {
|
| 471 | smp_wmb();
|
| 472 | pmd_populate_kernel(&init_mm, pmd, new);
|
| 473 | new = NULL;
|
| 474 | }
|
| 475 | spin_unlock(&init_mm.page_table_lock);
|
| 476 | if (new)
|
| 477 | pte_free_kernel(&init_mm, new);
|
| 478 | return 0;
|
| 479 | }
|
| 480 |
|
| 481 | static inline void init_rss_vec(int *rss)
|
| 482 | {
|
| 483 | memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
|
| 484 | }
|
| 485 |
|
| 486 | static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
|
| 487 | {
|
| 488 | int i;
|
| 489 |
|
| 490 | for (i = 0; i < NR_MM_COUNTERS; i++)
|
| 491 | if (rss[i])
|
| 492 | add_mm_counter(mm, i, rss[i]);
|
| 493 | }
|
| 494 |
|
| 495 | static bool is_bad_page_map_ratelimited(void)
|
| 496 | {
|
| 497 | static unsigned long resume;
|
| 498 | static unsigned long nr_shown;
|
| 499 | static unsigned long nr_unshown;
|
| 500 |
|
| 501 | |
| 502 | * Allow a burst of 60 reports, then keep quiet for that minute;
|
| 503 | * or allow a steady drip of one report per second.
|
| 504 |
|
| 505 | if (nr_shown == 60) {
|
| 506 | if (time_before(jiffies, resume)) {
|
| 507 | nr_unshown++;
|
| 508 | return true;
|
| 509 | }
|
| 510 | if (nr_unshown) {
|
| 511 | pr_alert("BUG: Bad page map: %lu messages suppressed\n",
|
| 512 | nr_unshown);
|
| 513 | nr_unshown = 0;
|
| 514 | }
|
| 515 | nr_shown = 0;
|
| 516 | }
|
| 517 | if (nr_shown++ == 0)
|
| 518 | resume = jiffies + 60 * HZ;
|
| 519 | return false;
|
| 520 | }
|
| 521 |
|
| 522 | static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size)
|
| 523 | {
|
| 524 | if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) {
|
| 525 | snprintf(buf, buf_size, "overflow");
|
| 526 | return;
|
| 527 | }
|
| 528 |
|
| 529 | switch (entry_size) {
|
| 530 | case sizeof(u32):
|
| 531 | snprintf(buf, buf_size, "%08x", *(const u32 *)entry);
|
| 532 | break;
|
| 533 | case sizeof(u64):
|
| 534 | snprintf(buf, buf_size, "%016llx", *(const u64 *)entry);
|
| 535 | break;
|
| 536 | #if defined(__SIZEOF_INT128__)
|
| 537 | case sizeof(u128):
|
| 538 | snprintf(buf, buf_size, "%016llx%016llx",
|
| 539 | (unsigned long long)(*(const u128 *)entry >> 64),
|
| 540 | (unsigned long long)*(const u128 *)entry);
|
| 541 | break;
|
| 542 | #endif
|
| 543 | default:
|
| 544 | snprintf(buf, buf_size, "unsupported");
|
| 545 | break;
|
| 546 | }
|
| 547 | }
|
| 548 |
|
| 549 | #define ptval_to_str(buf, val) \
|
| 550 | do { \
|
| 551 | auto __val = (val); \
|
| 552 | \
|
| 553 | ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \
|
| 554 | } while (0)
|
| 555 |
|
| 556 | #if defined(__SIZEOF_INT128__)
|
| 557 | #define PTVAL_STR_MAX (32 + 1)
|
| 558 | #else
|
| 559 | #define PTVAL_STR_MAX (16 + 1)
|
| 560 | #endif
|
| 561 |
|
| 562 | static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr)
|
| 563 | {
|
| 564 | char pgd_str[PTVAL_STR_MAX];
|
| 565 | char p4d_str[PTVAL_STR_MAX];
|
| 566 | char pud_str[PTVAL_STR_MAX];
|
| 567 | char pmd_str[PTVAL_STR_MAX];
|
| 568 | p4d_t p4d, *p4dp;
|
| 569 | pud_t pud, *pudp;
|
| 570 | pmd_t pmd, *pmdp;
|
| 571 | pgd_t *pgdp;
|
| 572 |
|
| 573 | |
| 574 | * Although this looks like a fully lockless pgtable walk, it is not:
|
| 575 | * see locking requirements for print_bad_page_map().
|
| 576 |
|
| 577 | pgdp = pgd_offset(mm, addr);
|
| 578 | ptval_to_str(pgd_str, pgd_val(*pgdp));
|
| 579 |
|
| 580 | if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) {
|
| 581 | pr_alert("pgd:%s\n", pgd_str);
|
| 582 | return;
|
| 583 | }
|
| 584 |
|
| 585 | p4dp = p4d_offset(pgdp, addr);
|
| 586 | p4d = p4dp_get(p4dp);
|
| 587 | ptval_to_str(p4d_str, p4d_val(p4d));
|
| 588 |
|
| 589 | if (!p4d_present(p4d) || p4d_leaf(p4d)) {
|
| 590 | pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str);
|
| 591 | return;
|
| 592 | }
|
| 593 |
|
| 594 | pudp = pud_offset(p4dp, addr);
|
| 595 | pud = pudp_get(pudp);
|
| 596 | ptval_to_str(pud_str, pud_val(pud));
|
| 597 |
|
| 598 | if (!pud_present(pud) || pud_leaf(pud)) {
|
| 599 | pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str);
|
| 600 | return;
|
| 601 | }
|
| 602 |
|
| 603 | pmdp = pmd_offset(pudp, addr);
|
| 604 | pmd = pmdp_get(pmdp);
|
| 605 | ptval_to_str(pmd_str, pmd_val(pmd));
|
| 606 |
|
| 607 | |
| 608 | * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE,
|
| 609 | * because the table should already be mapped by the caller and
|
| 610 | * doing another map would be bad. print_bad_page_map() should
|
| 611 | * already take care of printing the PTE.
|
| 612 |
|
| 613 | pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str);
|
| 614 | }
|
| 615 |
|
| 616 | |
| 617 | * This function is called to print an error when a bad page table entry (e.g.,
|
| 618 | * corrupted page table entry) is found. For example, we might have a
|
| 619 | * PFN-mapped pte in a region that doesn't allow it.
|
| 620 | *
|
| 621 | * The calling function must still handle the error.
|
| 622 | *
|
| 623 | * This function must be called during a proper page table walk, as it will
|
| 624 | * re-walk the page table to dump information: the caller MUST prevent page
|
| 625 | * table teardown (by holding mmap, vma or rmap lock) and MUST hold the leaf
|
| 626 | * page table lock.
|
| 627 |
|
| 628 | static void print_bad_page_map(struct vm_area_struct *vma,
|
| 629 | unsigned long addr, const void *entry, size_t entry_size,
|
| 630 | struct page *page, enum pgtable_level level)
|
| 631 | {
|
| 632 | struct address_space *mapping;
|
| 633 | char entry_str[PTVAL_STR_MAX];
|
| 634 | pgoff_t index, anon_index;
|
| 635 |
|
| 636 | if (is_bad_page_map_ratelimited())
|
| 637 | return;
|
| 638 |
|
| 639 | mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
|
| 640 | index = linear_page_index(vma, addr);
|
| 641 | anon_index = __linear_anon_page_index(vma, addr);
|
| 642 |
|
| 643 | ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size);
|
| 644 | pr_alert("BUG: Bad page map in process %s %s:%s", current->comm,
|
| 645 | pgtable_level_to_str(level), entry_str);
|
| 646 | __print_bad_page_map_pgtable(vma->vm_mm, addr);
|
| 647 | if (page)
|
| 648 | dump_page(page, "bad page map");
|
| 649 | pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px",
|
| 650 | (void *)addr, vma->vm_flags, vma->anon_vma, mapping);
|
| 651 | if (!vma_is_cow_mapping(vma) || index == anon_index) {
|
| 652 | pr_cont(" index:%lx\n", index);
|
| 653 | } else {
|
| 654 | pr_cont(" index:%lx (file) %lx (anon)\n", index, anon_index);
|
| 655 | }
|
| 656 |
|
| 657 | pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n",
|
| 658 | vma->vm_file,
|
| 659 | vma->vm_ops ? vma->vm_ops->fault : NULL,
|
| 660 | vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
|
| 661 | vma->vm_file ? vma->vm_file->f_op->mmap_prepare : NULL,
|
| 662 | mapping ? mapping->a_ops->read_folio : NULL);
|
| 663 | dump_stack();
|
| 664 | add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
|
| 665 | }
|
| 666 |
|
| 667 | static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
|
| 668 | {
|
| 669 | switch (level) {
|
| 670 | case PGTABLE_LEVEL_PTE:
|
| 671 | return IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL);
|
| 672 | case PGTABLE_LEVEL_PMD:
|
| 673 | return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP);
|
| 674 | case PGTABLE_LEVEL_PUD:
|
| 675 | return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP);
|
| 676 | default:
|
| 677 | return false;
|
| 678 | }
|
| 679 | }
|
| 680 |
|
| 681 | static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
|
| 682 | pte_t pte, struct page *page)
|
| 683 | {
|
| 684 | auto entry = pte_val(pte);
|
| 685 |
|
| 686 | print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE);
|
| 687 | }
|
| 688 |
|
| 689 | |
| 690 | * __vm_normal_page() - Get the "struct page" associated with a page table entry.
|
| 691 | * @vma: The VMA mapping the page table entry.
|
| 692 | * @addr: The address where the page table entry is mapped.
|
| 693 | * @pfn: The PFN stored in the page table entry.
|
| 694 | * @special: Whether the page table entry is marked "special".
|
| 695 | * @entry: The page table entry value for error reporting purposes only.
|
| 696 | * @entry_size: The size of @entry.
|
| 697 | * @level: The page table level for error reporting purposes only.
|
| 698 | *
|
| 699 | * "Special" mappings do not wish to be associated with a "struct page" (either
|
| 700 | * it doesn't exist, or it exists but they don't want to touch it). In this
|
| 701 | * case, NULL is returned here. "Normal" mappings do have a struct page and
|
| 702 | * are ordinarily refcounted.
|
| 703 | *
|
| 704 | * Page mappings of the shared zero folios are always considered "special", as
|
| 705 | * they are not ordinarily refcounted: neither the refcount nor the mapcount
|
| 706 | * of these folios is adjusted when mapping them into user page tables.
|
| 707 | * Selected page table walkers (such as GUP) can still identify mappings of the
|
| 708 | * shared zero folios and work with the underlying "struct page".
|
| 709 | *
|
| 710 | * There are 2 broad cases. Firstly, an architecture may define a "special"
|
| 711 | * page table entry bit, such as pte_special(), in which case this function is
|
| 712 | * trivial. Secondly, an architecture may not have a spare page table
|
| 713 | * entry bit, which requires a more complicated scheme, described below.
|
| 714 | *
|
| 715 | * With CONFIG_FIND_NORMAL_PAGE, we might have the "special" bit set on
|
| 716 | * page table entries that actually map "normal" pages: however, that page
|
| 717 | * cannot be looked up through the PFN stored in the page table entry, but
|
| 718 | * instead will be looked up through vm_ops->find_normal_page(). So far, this
|
| 719 | * only applies to PTEs.
|
| 720 | *
|
| 721 | * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
|
| 722 | * special mapping (even if there are underlying and valid "struct pages").
|
| 723 | * COWed pages of a VM_PFNMAP are always normal.
|
| 724 | *
|
| 725 | * The way we recognize COWed pages within VM_PFNMAP mappings is through the
|
| 726 | * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
|
| 727 | * set, and the vm_pgoff will point to the first PFN mapped: thus every special
|
| 728 | * mapping will always honor the rule
|
| 729 | *
|
| 730 | * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
|
| 731 | *
|
| 732 | * And for normal mappings this is false.
|
| 733 | *
|
| 734 | * This restricts such mappings to be a linear translation from virtual address
|
| 735 | * to pfn. To get around this restriction, we allow arbitrary mappings so long
|
| 736 | * as the vma is not a COW mapping; in that case, we know that all ptes are
|
| 737 | * special (because none can have been COWed).
|
| 738 | *
|
| 739 | *
|
| 740 | * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
|
| 741 | *
|
| 742 | * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
|
| 743 | * page" backing, however the difference is that _all_ pages with a struct
|
| 744 | * page (that is, those where pfn_valid is true, except the shared zero
|
| 745 | * folios) are refcounted and considered normal pages by the VM.
|
| 746 | *
|
| 747 | * The disadvantage is that pages are refcounted (which can be slower and
|
| 748 | * simply not an option for some PFNMAP users). The advantage is that we
|
| 749 | * don't have to follow the strict linearity rule of PFNMAP mappings in
|
| 750 | * order to support COWable mappings.
|
| 751 | *
|
| 752 | * Return: Returns the "struct page" if this is a "normal" mapping. Returns
|
| 753 | * NULL if this is a "special" mapping.
|
| 754 |
|
| 755 | static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
|
| 756 | unsigned long addr, unsigned long pfn, bool special,
|
| 757 | const void *entry, size_t entry_size, enum pgtable_level level)
|
| 758 | {
|
| 759 | if (pgtable_level_has_pxx_special(level)) {
|
| 760 | if (unlikely(special)) {
|
| 761 | #ifdef CONFIG_FIND_NORMAL_PAGE
|
| 762 | if (vma->vm_ops && vma->vm_ops->find_normal_page)
|
| 763 | return vma->vm_ops->find_normal_page(vma, addr);
|
| 764 | #endif
|
| 765 | if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
|
| 766 | return NULL;
|
| 767 | if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
|
| 768 | return NULL;
|
| 769 |
|
| 770 | print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
|
| 771 | return NULL;
|
| 772 | }
|
| 773 | |
| 774 | * With working pte_special()/pmd_special()..., any special page
|
| 775 | * table mappings (incl. shared zero folios) are marked
|
| 776 | * accordingly.
|
| 777 |
|
| 778 | } else {
|
| 779 | if (unlikely(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))) {
|
| 780 | if (vma->vm_flags & VM_MIXEDMAP) {
|
| 781 |
|
| 782 | if (!pfn_valid(pfn))
|
| 783 | return NULL;
|
| 784 | } else {
|
| 785 | const pgoff_t index = linear_page_index(vma, addr);
|
| 786 |
|
| 787 |
|
| 788 | if (pfn == index)
|
| 789 | return NULL;
|
| 790 | if (!vma_is_cow_mapping(vma))
|
| 791 | return NULL;
|
| 792 | }
|
| 793 | }
|
| 794 |
|
| 795 | if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
|
| 796 | return NULL;
|
| 797 | }
|
| 798 |
|
| 799 | if (unlikely(pfn > highest_memmap_pfn)) {
|
| 800 |
|
| 801 | print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
|
| 802 | return NULL;
|
| 803 | }
|
| 804 | |
| 805 | * NOTE! We still have PageReserved() pages in the page tables.
|
| 806 | * For example, VDSO mappings can cause them to exist.
|
| 807 |
|
| 808 | VM_WARN_ON_ONCE(is_zero_pfn(pfn) || is_huge_zero_pfn(pfn));
|
| 809 | return pfn_to_page(pfn);
|
| 810 | }
|
| 811 |
|
| 812 | |
| 813 | * vm_normal_page() - Get the "struct page" associated with a PTE
|
| 814 | * @vma: The VMA mapping the @pte.
|
| 815 | * @addr: The address where the @pte is mapped.
|
| 816 | * @pte: The PTE.
|
| 817 | *
|
| 818 | * Get the "struct page" associated with a PTE. See __vm_normal_page()
|
| 819 | * for details on "normal" and "special" mappings.
|
| 820 | *
|
| 821 | * Return: Returns the "struct page" if this is a "normal" mapping. Returns
|
| 822 | * NULL if this is a "special" mapping.
|
| 823 |
|
| 824 | struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
|
| 825 | pte_t pte)
|
| 826 | {
|
| 827 | auto entry = pte_val(pte);
|
| 828 |
|
| 829 | return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte),
|
| 830 | &entry, sizeof(entry), PGTABLE_LEVEL_PTE);
|
| 831 | }
|
| 832 |
|
| 833 | |
| 834 | * vm_normal_folio() - Get the "struct folio" associated with a PTE
|
| 835 | * @vma: The VMA mapping the @pte.
|
| 836 | * @addr: The address where the @pte is mapped.
|
| 837 | * @pte: The PTE.
|
| 838 | *
|
| 839 | * Get the "struct folio" associated with a PTE. See __vm_normal_page()
|
| 840 | * for details on "normal" and "special" mappings.
|
| 841 | *
|
| 842 | * Return: Returns the "struct folio" if this is a "normal" mapping. Returns
|
| 843 | * NULL if this is a "special" mapping.
|
| 844 |
|
| 845 | struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
|
| 846 | pte_t pte)
|
| 847 | {
|
| 848 | struct page *page = vm_normal_page(vma, addr, pte);
|
| 849 |
|
| 850 | if (page)
|
| 851 | return page_folio(page);
|
| 852 | return NULL;
|
| 853 | }
|
| 854 |
|
| 855 | #ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
|
| 856 | |
| 857 | * vm_normal_page_pmd() - Get the "struct page" associated with a PMD
|
| 858 | * @vma: The VMA mapping the @pmd.
|
| 859 | * @addr: The address where the @pmd is mapped.
|
| 860 | * @pmd: The PMD.
|
| 861 | *
|
| 862 | * Get the "struct page" associated with a PTE. See __vm_normal_page()
|
| 863 | * for details on "normal" and "special" mappings.
|
| 864 | *
|
| 865 | * Return: Returns the "struct page" if this is a "normal" mapping. Returns
|
| 866 | * NULL if this is a "special" mapping.
|
| 867 |
|
| 868 | struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
|
| 869 | pmd_t pmd)
|
| 870 | {
|
| 871 | auto entry = pmd_val(pmd);
|
| 872 |
|
| 873 | return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd),
|
| 874 | &entry, sizeof(entry), PGTABLE_LEVEL_PMD);
|
| 875 | }
|
| 876 |
|
| 877 | |
| 878 | * vm_normal_folio_pmd() - Get the "struct folio" associated with a PMD
|
| 879 | * @vma: The VMA mapping the @pmd.
|
| 880 | * @addr: The address where the @pmd is mapped.
|
| 881 | * @pmd: The PMD.
|
| 882 | *
|
| 883 | * Get the "struct folio" associated with a PTE. See __vm_normal_page()
|
| 884 | * for details on "normal" and "special" mappings.
|
| 885 | *
|
| 886 | * Return: Returns the "struct folio" if this is a "normal" mapping. Returns
|
| 887 | * NULL if this is a "special" mapping.
|
| 888 |
|
| 889 | struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
|
| 890 | unsigned long addr, pmd_t pmd)
|
| 891 | {
|
| 892 | struct page *page = vm_normal_page_pmd(vma, addr, pmd);
|
| 893 |
|
| 894 | if (page)
|
| 895 | return page_folio(page);
|
| 896 | return NULL;
|
| 897 | }
|
| 898 |
|
| 899 | |
| 900 | * vm_normal_page_pud() - Get the "struct page" associated with a PUD
|
| 901 | * @vma: The VMA mapping the @pud.
|
| 902 | * @addr: The address where the @pud is mapped.
|
| 903 | * @pud: The PUD.
|
| 904 | *
|
| 905 | * Get the "struct page" associated with a PUD. See __vm_normal_page()
|
| 906 | * for details on "normal" and "special" mappings.
|
| 907 | *
|
| 908 | * Return: Returns the "struct page" if this is a "normal" mapping. Returns
|
| 909 | * NULL if this is a "special" mapping.
|
| 910 |
|
| 911 | struct page *vm_normal_page_pud(struct vm_area_struct *vma,
|
| 912 | unsigned long addr, pud_t pud)
|
| 913 | {
|
| 914 | auto entry = pud_val(pud);
|
| 915 |
|
| 916 | return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud),
|
| 917 | &entry, sizeof(entry), PGTABLE_LEVEL_PUD);
|
| 918 | }
|
| 919 | #endif
|
| 920 |
|
| 921 | |
| 922 | * restore_exclusive_pte - Restore a device-exclusive entry
|
| 923 | * @vma: VMA covering @address
|
| 924 | * @folio: the mapped folio
|
| 925 | * @page: the mapped folio page
|
| 926 | * @address: the virtual address
|
| 927 | * @ptep: pte pointer into the locked page table mapping the folio page
|
| 928 | * @orig_pte: pte value at @ptep
|
| 929 | *
|
| 930 | * Restore a device-exclusive non-swap entry to an ordinary present pte.
|
| 931 | *
|
| 932 | * The folio and the page table must be locked, and MMU notifiers must have
|
| 933 | * been called to invalidate any (exclusive) device mappings.
|
| 934 | *
|
| 935 | * Locking the folio makes sure that anybody who just converted the pte to
|
| 936 | * a device-exclusive entry can map it into the device to make forward
|
| 937 | * progress without others converting it back until the folio was unlocked.
|
| 938 | *
|
| 939 | * If the folio lock ever becomes an issue, we can stop relying on the folio
|
| 940 | * lock; it might make some scenarios with heavy thrashing less likely to
|
| 941 | * make forward progress, but these scenarios might not be valid use cases.
|
| 942 | *
|
| 943 | * Note that the folio lock does not protect against all cases of concurrent
|
| 944 | * page table modifications (e.g., MADV_DONTNEED, mprotect), so device drivers
|
| 945 | * must use MMU notifiers to sync against any concurrent changes.
|
| 946 |
|
| 947 | static void restore_exclusive_pte(struct vm_area_struct *vma,
|
| 948 | struct folio *folio, struct page *page, unsigned long address,
|
| 949 | pte_t *ptep, pte_t orig_pte)
|
| 950 | {
|
| 951 | pte_t pte;
|
| 952 |
|
| 953 | VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 954 |
|
| 955 | pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
|
| 956 | if (pte_swp_soft_dirty(orig_pte))
|
| 957 | pte = pte_mksoft_dirty(pte);
|
| 958 |
|
| 959 | if (pte_swp_uffd(orig_pte))
|
| 960 | pte = pte_mkuffd(pte);
|
| 961 |
|
| 962 |
|
| 963 | if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma))
|
| 964 | pte = pte_modify(pte, PAGE_NONE);
|
| 965 |
|
| 966 | if ((vma->vm_flags & VM_WRITE) &&
|
| 967 | can_change_pte_writable(vma, address, pte)) {
|
| 968 | if (folio_test_dirty(folio))
|
| 969 | pte = pte_mkdirty(pte);
|
| 970 | pte = pte_mkwrite(pte, vma);
|
| 971 | }
|
| 972 | set_pte_at(vma->vm_mm, address, ptep, pte);
|
| 973 |
|
| 974 | |
| 975 | * No need to invalidate - it was non-present before. However
|
| 976 | * secondary CPUs may have mappings that need invalidating.
|
| 977 |
|
| 978 | update_mmu_cache(vma, address, ptep);
|
| 979 | }
|
| 980 |
|
| 981 | |
| 982 | * Tries to restore an exclusive pte if the page lock can be acquired without
|
| 983 | * sleeping.
|
| 984 |
|
| 985 | static int try_restore_exclusive_pte(struct vm_area_struct *vma,
|
| 986 | unsigned long addr, pte_t *ptep, pte_t orig_pte)
|
| 987 | {
|
| 988 | const softleaf_t entry = softleaf_from_pte(orig_pte);
|
| 989 | struct page *page = softleaf_to_page(entry);
|
| 990 | struct folio *folio = page_folio(page);
|
| 991 |
|
| 992 | if (folio_trylock(folio)) {
|
| 993 | restore_exclusive_pte(vma, folio, page, addr, ptep, orig_pte);
|
| 994 | folio_unlock(folio);
|
| 995 | return 0;
|
| 996 | }
|
| 997 |
|
| 998 | return -EBUSY;
|
| 999 | }
|
| 1000 |
|
| 1001 | |
| 1002 | * copy one vm_area from one task to the other. Assumes the page tables
|
| 1003 | * already present in the new task to be cleared in the whole range
|
| 1004 | * covered by this vma.
|
| 1005 |
|
| 1006 |
|
| 1007 | static unsigned long
|
| 1008 | copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
|
| 1009 | pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
|
| 1010 | struct vm_area_struct *src_vma, unsigned long addr, int *rss)
|
| 1011 | {
|
| 1012 | pte_t orig_pte = ptep_get(src_pte);
|
| 1013 | softleaf_t entry = softleaf_from_pte(orig_pte);
|
| 1014 | pte_t pte = orig_pte;
|
| 1015 | struct folio *folio;
|
| 1016 | struct page *page;
|
| 1017 |
|
| 1018 | if (likely(softleaf_is_swap(entry))) {
|
| 1019 | if (swap_dup_entry_direct(entry) < 0)
|
| 1020 | return -EIO;
|
| 1021 |
|
| 1022 | mm_prepare_for_swap_entries(dst_mm);
|
| 1023 |
|
| 1024 | if (pte_swp_exclusive(orig_pte)) {
|
| 1025 | pte = pte_swp_clear_exclusive(orig_pte);
|
| 1026 | set_pte_at(src_mm, addr, src_pte, pte);
|
| 1027 | }
|
| 1028 | rss[MM_SWAPENTS]++;
|
| 1029 | } else if (softleaf_is_migration(entry)) {
|
| 1030 | folio = softleaf_to_folio(entry);
|
| 1031 |
|
| 1032 | rss[mm_counter(folio)]++;
|
| 1033 |
|
| 1034 | if (!softleaf_is_migration_read(entry) &&
|
| 1035 | vma_is_cow_mapping(dst_vma)) {
|
| 1036 | |
| 1037 | * COW mappings require pages in both parent and child
|
| 1038 | * to be set to read. A previously exclusive entry is
|
| 1039 | * now shared.
|
| 1040 |
|
| 1041 | entry = make_readable_migration_entry(
|
| 1042 | swp_offset(entry));
|
| 1043 | pte = softleaf_to_pte(entry);
|
| 1044 | if (pte_swp_soft_dirty(orig_pte))
|
| 1045 | pte = pte_swp_mksoft_dirty(pte);
|
| 1046 | if (pte_swp_uffd(orig_pte))
|
| 1047 | pte = pte_swp_mkuffd(pte);
|
| 1048 | set_pte_at(src_mm, addr, src_pte, pte);
|
| 1049 | }
|
| 1050 | } else if (softleaf_is_device_private(entry)) {
|
| 1051 | page = softleaf_to_page(entry);
|
| 1052 | folio = page_folio(page);
|
| 1053 |
|
| 1054 | |
| 1055 | * Update rss count even for unaddressable pages, as
|
| 1056 | * they should treated just like normal pages in this
|
| 1057 | * respect.
|
| 1058 | *
|
| 1059 | * We will likely want to have some new rss counters
|
| 1060 | * for unaddressable pages, at some point. But for now
|
| 1061 | * keep things as they are.
|
| 1062 |
|
| 1063 | folio_get(folio);
|
| 1064 | rss[mm_counter(folio)]++;
|
| 1065 |
|
| 1066 | folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma);
|
| 1067 |
|
| 1068 | |
| 1069 | * We do not preserve soft-dirty information, because so
|
| 1070 | * far, checkpoint/restore is the only feature that
|
| 1071 | * requires that. And checkpoint/restore does not work
|
| 1072 | * when a device driver is involved (you cannot easily
|
| 1073 | * save and restore device driver state).
|
| 1074 |
|
| 1075 | if (softleaf_is_device_private_write(entry) &&
|
| 1076 | vma_is_cow_mapping(dst_vma)) {
|
| 1077 | entry = make_readable_device_private_entry(
|
| 1078 | swp_offset(entry));
|
| 1079 | pte = swp_entry_to_pte(entry);
|
| 1080 | if (pte_swp_uffd(orig_pte))
|
| 1081 | pte = pte_swp_mkuffd(pte);
|
| 1082 | set_pte_at(src_mm, addr, src_pte, pte);
|
| 1083 | }
|
| 1084 | } else if (softleaf_is_device_exclusive(entry)) {
|
| 1085 | |
| 1086 | * Make device exclusive entries present by restoring the
|
| 1087 | * original entry then copying as for a present pte. Device
|
| 1088 | * exclusive entries currently only support private writable
|
| 1089 | * (ie. COW) mappings.
|
| 1090 |
|
| 1091 | VM_BUG_ON(!vma_is_cow_mapping(src_vma));
|
| 1092 | if (try_restore_exclusive_pte(src_vma, addr, src_pte, orig_pte))
|
| 1093 | return -EBUSY;
|
| 1094 | return -ENOENT;
|
| 1095 | } else if (softleaf_is_marker(entry)) {
|
| 1096 | pte_marker marker = copy_pte_marker(entry, dst_vma);
|
| 1097 |
|
| 1098 | if (marker)
|
| 1099 | set_pte_at(dst_mm, addr, dst_pte,
|
| 1100 | make_pte_marker(marker));
|
| 1101 | return 0;
|
| 1102 | }
|
| 1103 | if (!userfaultfd_protected(dst_vma))
|
| 1104 | pte = pte_swp_clear_uffd(pte);
|
| 1105 | set_pte_at(dst_mm, addr, dst_pte, pte);
|
| 1106 | return 0;
|
| 1107 | }
|
| 1108 |
|
| 1109 | |
| 1110 | * Copy a present and normal page.
|
| 1111 | *
|
| 1112 | * NOTE! The usual case is that this isn't required;
|
| 1113 | * instead, the caller can just increase the page refcount
|
| 1114 | * and re-use the pte the traditional way.
|
| 1115 | *
|
| 1116 | * And if we need a pre-allocated page but don't yet have
|
| 1117 | * one, return a negative error to let the preallocation
|
| 1118 | * code know so that it can do so outside the page table
|
| 1119 | * lock.
|
| 1120 |
|
| 1121 | static inline int
|
| 1122 | copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
| 1123 | pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
|
| 1124 | struct folio **prealloc, struct page *page)
|
| 1125 | {
|
| 1126 | struct folio *new_folio;
|
| 1127 | pte_t pte;
|
| 1128 |
|
| 1129 | new_folio = *prealloc;
|
| 1130 | if (!new_folio)
|
| 1131 | return -EAGAIN;
|
| 1132 |
|
| 1133 | |
| 1134 | * We have a prealloc page, all good! Take it
|
| 1135 | * over and copy the page & arm it.
|
| 1136 |
|
| 1137 |
|
| 1138 | if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma))
|
| 1139 | return -EHWPOISON;
|
| 1140 |
|
| 1141 | *prealloc = NULL;
|
| 1142 | __folio_mark_uptodate(new_folio);
|
| 1143 | folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE);
|
| 1144 | folio_add_lru_vma(new_folio, dst_vma);
|
| 1145 | rss[MM_ANONPAGES]++;
|
| 1146 |
|
| 1147 |
|
| 1148 | pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot);
|
| 1149 | pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
|
| 1150 | if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) {
|
| 1151 |
|
| 1152 | pte = pte_mkuffd(pte);
|
| 1153 |
|
| 1154 | if (userfaultfd_rwp(dst_vma))
|
| 1155 | pte = pte_modify(pte, PAGE_NONE);
|
| 1156 | }
|
| 1157 | set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
|
| 1158 | return 0;
|
| 1159 | }
|
| 1160 |
|
| 1161 | static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
|
| 1162 | struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte,
|
| 1163 | pte_t pte, unsigned long addr, int nr)
|
| 1164 | {
|
| 1165 | struct mm_struct *src_mm = src_vma->vm_mm;
|
| 1166 | bool writable;
|
| 1167 |
|
| 1168 | |
| 1169 | * Snapshot writability before the RWP-disarm rewrite below: when the
|
| 1170 | * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can
|
| 1171 | * silently drop _PAGE_RW from a resolved (no-marker) writable PTE,
|
| 1172 | * so a later pte_write(pte) check would skip the COW wrprotect and
|
| 1173 | * leave the parent writable over a folio shared with the child.
|
| 1174 |
|
| 1175 | writable = pte_write(pte);
|
| 1176 |
|
| 1177 | |
| 1178 | * Child is not RWP-armed: restore accessible protection so the
|
| 1179 | * inherited PAGE_NONE does not cost a fault on first read. Gate on
|
| 1180 | * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing)
|
| 1181 | * are not normalised away.
|
| 1182 |
|
| 1183 | if (!userfaultfd_protected(dst_vma)) {
|
| 1184 | if (userfaultfd_rwp(src_vma) && pte_uffd(pte))
|
| 1185 | pte = pte_modify(pte, dst_vma->vm_page_prot);
|
| 1186 | pte = pte_clear_uffd(pte);
|
| 1187 | }
|
| 1188 |
|
| 1189 |
|
| 1190 | if (vma_is_cow_mapping(src_vma) && writable) {
|
| 1191 | wrprotect_ptes(src_mm, addr, src_pte, nr);
|
| 1192 | pte = pte_wrprotect(pte);
|
| 1193 | }
|
| 1194 |
|
| 1195 |
|
| 1196 | if (src_vma->vm_flags & VM_SHARED)
|
| 1197 | pte = pte_mkclean(pte);
|
| 1198 | pte = pte_mkold(pte);
|
| 1199 |
|
| 1200 | set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
|
| 1201 | }
|
| 1202 |
|
| 1203 | |
| 1204 | * Copy one present PTE, trying to batch-process subsequent PTEs that map
|
| 1205 | * consecutive pages of the same folio by copying them as well.
|
| 1206 | *
|
| 1207 | * Returns -EAGAIN if one preallocated page is required to copy the next PTE.
|
| 1208 | * Otherwise, returns the number of copied PTEs (at least 1).
|
| 1209 |
|
| 1210 | static inline int
|
| 1211 | copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
| 1212 | pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr,
|
| 1213 | int max_nr, int *rss, struct folio **prealloc)
|
| 1214 | {
|
| 1215 | fpb_t flags = FPB_MERGE_WRITE;
|
| 1216 | struct page *page;
|
| 1217 | struct folio *folio;
|
| 1218 | int err, nr;
|
| 1219 |
|
| 1220 | page = vm_normal_page(src_vma, addr, pte);
|
| 1221 | if (unlikely(!page))
|
| 1222 | goto copy_pte;
|
| 1223 |
|
| 1224 | folio = page_folio(page);
|
| 1225 |
|
| 1226 | |
| 1227 | * If we likely have to copy, just don't bother with batching. Make
|
| 1228 | * sure that the common "small folio" case is as fast as possible
|
| 1229 | * by keeping the batching logic separate.
|
| 1230 |
|
| 1231 | if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {
|
| 1232 | if (!(src_vma->vm_flags & VM_SHARED))
|
| 1233 | flags |= FPB_RESPECT_DIRTY;
|
| 1234 | if (vma_soft_dirty_enabled(src_vma))
|
| 1235 | flags |= FPB_RESPECT_SOFT_DIRTY;
|
| 1236 |
|
| 1237 | nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags);
|
| 1238 | folio_ref_add(folio, nr);
|
| 1239 | if (folio_test_anon(folio)) {
|
| 1240 | if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
|
| 1241 | nr, dst_vma, src_vma))) {
|
| 1242 | folio_ref_sub(folio, nr);
|
| 1243 | return -EAGAIN;
|
| 1244 | }
|
| 1245 | rss[MM_ANONPAGES] += nr;
|
| 1246 | VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
|
| 1247 | } else {
|
| 1248 | folio_dup_file_rmap_ptes(folio, page, nr, dst_vma);
|
| 1249 | rss[mm_counter_file(folio)] += nr;
|
| 1250 | }
|
| 1251 | __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
|
| 1252 | addr, nr);
|
| 1253 | return nr;
|
| 1254 | }
|
| 1255 |
|
| 1256 | folio_get(folio);
|
| 1257 | if (folio_test_anon(folio)) {
|
| 1258 | |
| 1259 | * If this page may have been pinned by the parent process,
|
| 1260 | * copy the page immediately for the child so that we'll always
|
| 1261 | * guarantee the pinned page won't be randomly replaced in the
|
| 1262 | * future.
|
| 1263 |
|
| 1264 | if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) {
|
| 1265 |
|
| 1266 | folio_put(folio);
|
| 1267 | err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
|
| 1268 | addr, rss, prealloc, page);
|
| 1269 | return err ? err : 1;
|
| 1270 | }
|
| 1271 | rss[MM_ANONPAGES]++;
|
| 1272 | VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
|
| 1273 | } else {
|
| 1274 | folio_dup_file_rmap_pte(folio, page, dst_vma);
|
| 1275 | rss[mm_counter_file(folio)]++;
|
| 1276 | }
|
| 1277 |
|
| 1278 | copy_pte:
|
| 1279 | __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
|
| 1280 | return 1;
|
| 1281 | }
|
| 1282 |
|
| 1283 | static inline struct folio *folio_prealloc(struct mm_struct *src_mm,
|
| 1284 | struct vm_area_struct *vma, unsigned long addr, bool need_zero)
|
| 1285 | {
|
| 1286 | struct folio *new_folio;
|
| 1287 |
|
| 1288 | if (need_zero)
|
| 1289 | new_folio = vma_alloc_zeroed_movable_folio(vma, addr);
|
| 1290 | else
|
| 1291 | new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr);
|
| 1292 |
|
| 1293 | if (!new_folio)
|
| 1294 | return NULL;
|
| 1295 |
|
| 1296 | if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
|
| 1297 | folio_put(new_folio);
|
| 1298 | return NULL;
|
| 1299 | }
|
| 1300 | folio_throttle_swaprate(new_folio, GFP_KERNEL);
|
| 1301 |
|
| 1302 | return new_folio;
|
| 1303 | }
|
| 1304 |
|
| 1305 | static int
|
| 1306 | copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
| 1307 | pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
|
| 1308 | unsigned long end)
|
| 1309 | {
|
| 1310 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 1311 | struct mm_struct *src_mm = src_vma->vm_mm;
|
| 1312 | pte_t *orig_src_pte, *orig_dst_pte;
|
| 1313 | pte_t *src_pte, *dst_pte;
|
| 1314 | pmd_t dummy_pmdval;
|
| 1315 | pte_t ptent;
|
| 1316 | spinlock_t *src_ptl, *dst_ptl;
|
| 1317 | int progress, max_nr, ret = 0;
|
| 1318 | int rss[NR_MM_COUNTERS];
|
| 1319 | softleaf_t entry = softleaf_mk_none();
|
| 1320 | struct folio *prealloc = NULL;
|
| 1321 | int nr;
|
| 1322 |
|
| 1323 | again:
|
| 1324 | progress = 0;
|
| 1325 | init_rss_vec(rss);
|
| 1326 |
|
| 1327 | |
| 1328 | * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
|
| 1329 | * error handling here, assume that exclusive mmap_lock on dst and src
|
| 1330 | * protects anon from unexpected THP transitions; with shmem and file
|
| 1331 | * protected by mmap_lock-less collapse skipping areas with anon_vma
|
| 1332 | * (whereas vma_needs_copy() skips areas without anon_vma). A rework
|
| 1333 | * can remove such assumptions later, but this is good enough for now.
|
| 1334 |
|
| 1335 | dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
|
| 1336 | if (!dst_pte) {
|
| 1337 | ret = -ENOMEM;
|
| 1338 | goto out;
|
| 1339 | }
|
| 1340 |
|
| 1341 | |
| 1342 | * We already hold the exclusive mmap_lock, the copy_pte_range() and
|
| 1343 | * retract_page_tables() are using vma->anon_vma to be exclusive, so
|
| 1344 | * the PTE page is stable, and there is no need to get pmdval and do
|
| 1345 | * pmd_same() check.
|
| 1346 |
|
| 1347 | src_pte = pte_offset_map_rw_nolock(src_mm, src_pmd, addr, &dummy_pmdval,
|
| 1348 | &src_ptl);
|
| 1349 | if (!src_pte) {
|
| 1350 | pte_unmap_unlock(dst_pte, dst_ptl);
|
| 1351 |
|
| 1352 | goto out;
|
| 1353 | }
|
| 1354 | spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
|
| 1355 | orig_src_pte = src_pte;
|
| 1356 | orig_dst_pte = dst_pte;
|
| 1357 | lazy_mmu_mode_enable();
|
| 1358 |
|
| 1359 | do {
|
| 1360 | nr = 1;
|
| 1361 |
|
| 1362 | |
| 1363 | * We are holding two locks at this point - either of them
|
| 1364 | * could generate latencies in another task on another CPU.
|
| 1365 |
|
| 1366 | if (progress >= 32) {
|
| 1367 | progress = 0;
|
| 1368 | if (need_resched() ||
|
| 1369 | spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
|
| 1370 | break;
|
| 1371 | }
|
| 1372 | ptent = ptep_get(src_pte);
|
| 1373 | if (pte_none(ptent)) {
|
| 1374 | progress++;
|
| 1375 | continue;
|
| 1376 | }
|
| 1377 | if (unlikely(!pte_present(ptent))) {
|
| 1378 | ret = copy_nonpresent_pte(dst_mm, src_mm,
|
| 1379 | dst_pte, src_pte,
|
| 1380 | dst_vma, src_vma,
|
| 1381 | addr, rss);
|
| 1382 | if (ret == -EIO) {
|
| 1383 | entry = softleaf_from_pte(ptep_get(src_pte));
|
| 1384 | break;
|
| 1385 | } else if (ret == -EBUSY) {
|
| 1386 | break;
|
| 1387 | } else if (!ret) {
|
| 1388 | progress += 8;
|
| 1389 | continue;
|
| 1390 | }
|
| 1391 | ptent = ptep_get(src_pte);
|
| 1392 | VM_WARN_ON_ONCE(!pte_present(ptent));
|
| 1393 |
|
| 1394 | |
| 1395 | * Device exclusive entry restored, continue by copying
|
| 1396 | * the now present pte.
|
| 1397 |
|
| 1398 | WARN_ON_ONCE(ret != -ENOENT);
|
| 1399 | }
|
| 1400 |
|
| 1401 | max_nr = (end - addr) / PAGE_SIZE;
|
| 1402 | ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
|
| 1403 | ptent, addr, max_nr, rss, &prealloc);
|
| 1404 | |
| 1405 | * If we need a pre-allocated page for this pte, drop the
|
| 1406 | * locks, allocate, and try again.
|
| 1407 | * If copy failed due to hwpoison in source page, break out.
|
| 1408 |
|
| 1409 | if (unlikely(ret == -EAGAIN || ret == -EHWPOISON))
|
| 1410 | break;
|
| 1411 | if (unlikely(prealloc)) {
|
| 1412 | |
| 1413 | * pre-alloc page cannot be reused by next time so as
|
| 1414 | * to strictly follow mempolicy (e.g., alloc_page_vma()
|
| 1415 | * will allocate page according to address). This
|
| 1416 | * could only happen if one pinned pte changed.
|
| 1417 |
|
| 1418 | folio_put(prealloc);
|
| 1419 | prealloc = NULL;
|
| 1420 | }
|
| 1421 | nr = ret;
|
| 1422 | progress += 8 * nr;
|
| 1423 | } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr,
|
| 1424 | addr != end);
|
| 1425 |
|
| 1426 | lazy_mmu_mode_disable();
|
| 1427 | pte_unmap_unlock(orig_src_pte, src_ptl);
|
| 1428 | add_mm_rss_vec(dst_mm, rss);
|
| 1429 | pte_unmap_unlock(orig_dst_pte, dst_ptl);
|
| 1430 | cond_resched();
|
| 1431 |
|
| 1432 | if (ret == -EIO) {
|
| 1433 | VM_WARN_ON_ONCE(!entry.val);
|
| 1434 | if (swap_retry_table_alloc(entry, GFP_KERNEL) < 0) {
|
| 1435 | ret = -ENOMEM;
|
| 1436 | goto out;
|
| 1437 | }
|
| 1438 | entry.val = 0;
|
| 1439 | } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) {
|
| 1440 | goto out;
|
| 1441 | } else if (ret == -EAGAIN) {
|
| 1442 | prealloc = folio_prealloc(src_mm, src_vma, addr, false);
|
| 1443 | if (!prealloc)
|
| 1444 | return -ENOMEM;
|
| 1445 | } else if (ret < 0) {
|
| 1446 | VM_WARN_ON_ONCE(1);
|
| 1447 | }
|
| 1448 |
|
| 1449 |
|
| 1450 | ret = 0;
|
| 1451 |
|
| 1452 | if (addr != end)
|
| 1453 | goto again;
|
| 1454 | out:
|
| 1455 | if (unlikely(prealloc))
|
| 1456 | folio_put(prealloc);
|
| 1457 | return ret;
|
| 1458 | }
|
| 1459 |
|
| 1460 | static inline int
|
| 1461 | copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
| 1462 | pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
|
| 1463 | unsigned long end)
|
| 1464 | {
|
| 1465 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 1466 | struct mm_struct *src_mm = src_vma->vm_mm;
|
| 1467 | pmd_t *src_pmd, *dst_pmd;
|
| 1468 | unsigned long next;
|
| 1469 |
|
| 1470 | dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
|
| 1471 | if (!dst_pmd)
|
| 1472 | return -ENOMEM;
|
| 1473 | src_pmd = pmd_offset(src_pud, addr);
|
| 1474 | do {
|
| 1475 | next = pmd_addr_end(addr, end);
|
| 1476 | if (pmd_is_huge(*src_pmd)) {
|
| 1477 | int err;
|
| 1478 |
|
| 1479 | VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
|
| 1480 | err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
|
| 1481 | addr, dst_vma, src_vma);
|
| 1482 | if (err == -ENOMEM)
|
| 1483 | return -ENOMEM;
|
| 1484 | if (!err)
|
| 1485 | continue;
|
| 1486 |
|
| 1487 | }
|
| 1488 | if (pmd_none_or_clear_bad(src_pmd))
|
| 1489 | continue;
|
| 1490 | if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
|
| 1491 | addr, next))
|
| 1492 | return -ENOMEM;
|
| 1493 | } while (dst_pmd++, src_pmd++, addr = next, addr != end);
|
| 1494 | return 0;
|
| 1495 | }
|
| 1496 |
|
| 1497 | static inline int
|
| 1498 | copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
| 1499 | p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
|
| 1500 | unsigned long end)
|
| 1501 | {
|
| 1502 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 1503 | struct mm_struct *src_mm = src_vma->vm_mm;
|
| 1504 | pud_t *src_pud, *dst_pud;
|
| 1505 | unsigned long next;
|
| 1506 |
|
| 1507 | dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
|
| 1508 | if (!dst_pud)
|
| 1509 | return -ENOMEM;
|
| 1510 | src_pud = pud_offset(src_p4d, addr);
|
| 1511 | do {
|
| 1512 | next = pud_addr_end(addr, end);
|
| 1513 | if (pud_trans_huge(*src_pud)) {
|
| 1514 | int err;
|
| 1515 |
|
| 1516 | VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
|
| 1517 | err = copy_huge_pud(dst_mm, src_mm,
|
| 1518 | dst_pud, src_pud, addr, src_vma);
|
| 1519 | if (err == -ENOMEM)
|
| 1520 | return -ENOMEM;
|
| 1521 | if (!err)
|
| 1522 | continue;
|
| 1523 |
|
| 1524 | }
|
| 1525 | if (pud_none_or_clear_bad(src_pud))
|
| 1526 | continue;
|
| 1527 | if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
|
| 1528 | addr, next))
|
| 1529 | return -ENOMEM;
|
| 1530 | } while (dst_pud++, src_pud++, addr = next, addr != end);
|
| 1531 | return 0;
|
| 1532 | }
|
| 1533 |
|
| 1534 | static inline int
|
| 1535 | copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
|
| 1536 | pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
|
| 1537 | unsigned long end)
|
| 1538 | {
|
| 1539 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 1540 | p4d_t *src_p4d, *dst_p4d;
|
| 1541 | unsigned long next;
|
| 1542 |
|
| 1543 | dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
|
| 1544 | if (!dst_p4d)
|
| 1545 | return -ENOMEM;
|
| 1546 | src_p4d = p4d_offset(src_pgd, addr);
|
| 1547 | do {
|
| 1548 | next = p4d_addr_end(addr, end);
|
| 1549 | if (p4d_none_or_clear_bad(src_p4d))
|
| 1550 | continue;
|
| 1551 | if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
|
| 1552 | addr, next))
|
| 1553 | return -ENOMEM;
|
| 1554 | } while (dst_p4d++, src_p4d++, addr = next, addr != end);
|
| 1555 | return 0;
|
| 1556 | }
|
| 1557 |
|
| 1558 | |
| 1559 | * Return true if the vma needs to copy the pgtable during this fork(). Return
|
| 1560 | * false when we can speed up fork() by allowing lazy page faults later until
|
| 1561 | * when the child accesses the memory range.
|
| 1562 |
|
| 1563 | static bool
|
| 1564 | vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
|
| 1565 | {
|
| 1566 | |
| 1567 | * We check against dst_vma as while sane VMA flags will have been
|
| 1568 | * copied, VM_UFFD_WP may be set only on dst_vma.
|
| 1569 |
|
| 1570 | if (dst_vma->vm_flags & VM_COPY_ON_FORK)
|
| 1571 | return true;
|
| 1572 | |
| 1573 | * The presence of an anon_vma indicates an anonymous VMA has page
|
| 1574 | * tables which naturally cannot be reconstituted on page fault.
|
| 1575 |
|
| 1576 | if (src_vma->anon_vma)
|
| 1577 | return true;
|
| 1578 |
|
| 1579 | |
| 1580 | * Don't copy ptes where a page fault will fill them correctly. Fork
|
| 1581 | * becomes much lighter when there are big shared or private readonly
|
| 1582 | * mappings. The tradeoff is that copy_page_range is more efficient
|
| 1583 | * than faulting.
|
| 1584 |
|
| 1585 | return false;
|
| 1586 | }
|
| 1587 |
|
| 1588 | int
|
| 1589 | copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
|
| 1590 | {
|
| 1591 | pgd_t *src_pgd, *dst_pgd;
|
| 1592 | unsigned long addr = src_vma->vm_start;
|
| 1593 | unsigned long end = src_vma->vm_end;
|
| 1594 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 1595 | struct mm_struct *src_mm = src_vma->vm_mm;
|
| 1596 | struct mmu_notifier_range range;
|
| 1597 | unsigned long next;
|
| 1598 | bool is_cow;
|
| 1599 | int ret;
|
| 1600 |
|
| 1601 | if (!vma_needs_copy(dst_vma, src_vma))
|
| 1602 | return 0;
|
| 1603 |
|
| 1604 | if (is_vm_hugetlb_page(src_vma))
|
| 1605 | return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
|
| 1606 |
|
| 1607 | |
| 1608 | * We need to invalidate the secondary MMU mappings only when
|
| 1609 | * there could be a permission downgrade on the ptes of the
|
| 1610 | * parent mm. And a permission downgrade will only happen if
|
| 1611 | * vma_is_cow_mapping() returns true.
|
| 1612 |
|
| 1613 | is_cow = vma_is_cow_mapping(src_vma);
|
| 1614 |
|
| 1615 | if (is_cow) {
|
| 1616 | mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
|
| 1617 | 0, src_mm, addr, end);
|
| 1618 | mmu_notifier_invalidate_range_start(&range);
|
| 1619 | |
| 1620 | * Disabling preemption is not needed for the write side, as
|
| 1621 | * the read side doesn't spin, but goes to the mmap_lock.
|
| 1622 | *
|
| 1623 | * Use the raw variant of the seqcount_t write API to avoid
|
| 1624 | * lockdep complaining about preemptibility.
|
| 1625 |
|
| 1626 | vma_assert_write_locked(src_vma);
|
| 1627 | raw_write_seqcount_begin(&src_mm->write_protect_seq);
|
| 1628 | }
|
| 1629 |
|
| 1630 | ret = 0;
|
| 1631 | dst_pgd = pgd_offset(dst_mm, addr);
|
| 1632 | src_pgd = pgd_offset(src_mm, addr);
|
| 1633 | do {
|
| 1634 | next = pgd_addr_end(addr, end);
|
| 1635 | if (pgd_none_or_clear_bad(src_pgd))
|
| 1636 | continue;
|
| 1637 | if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
|
| 1638 | addr, next))) {
|
| 1639 | ret = -ENOMEM;
|
| 1640 | break;
|
| 1641 | }
|
| 1642 | } while (dst_pgd++, src_pgd++, addr = next, addr != end);
|
| 1643 |
|
| 1644 | if (is_cow) {
|
| 1645 | raw_write_seqcount_end(&src_mm->write_protect_seq);
|
| 1646 | mmu_notifier_invalidate_range_end(&range);
|
| 1647 | }
|
| 1648 | return ret;
|
| 1649 | }
|
| 1650 |
|
| 1651 |
|
| 1652 | static inline bool should_zap_cows(struct zap_details *details)
|
| 1653 | {
|
| 1654 |
|
| 1655 | if (!details)
|
| 1656 | return true;
|
| 1657 |
|
| 1658 | VM_WARN_ON_ONCE(details->skip_cows && details->reclaim_pt);
|
| 1659 |
|
| 1660 |
|
| 1661 | return !details->skip_cows;
|
| 1662 | }
|
| 1663 |
|
| 1664 |
|
| 1665 | static inline bool should_zap_folio(struct zap_details *details,
|
| 1666 | struct folio *folio)
|
| 1667 | {
|
| 1668 |
|
| 1669 | if (should_zap_cows(details))
|
| 1670 | return true;
|
| 1671 |
|
| 1672 |
|
| 1673 | return !folio_test_anon(folio);
|
| 1674 | }
|
| 1675 |
|
| 1676 | static inline bool zap_drop_markers(struct zap_details *details)
|
| 1677 | {
|
| 1678 | if (!details)
|
| 1679 | return false;
|
| 1680 |
|
| 1681 | return details->zap_flags & ZAP_FLAG_DROP_MARKER;
|
| 1682 | }
|
| 1683 |
|
| 1684 | |
| 1685 | * cond_install_uffd_wp_ptes - install uffd-wp markers after clearing PTEs
|
| 1686 | * @vma: The VMA the pages are mapped into.
|
| 1687 | * @addr: Address the first page of this batch is mapped at.
|
| 1688 | * @ptep: Page table pointer for the first entry of this batch.
|
| 1689 | * @pte: Old value of the entry pointed to by @ptep.
|
| 1690 | * @nr_ptes: Number of entries to install.
|
| 1691 | *
|
| 1692 | * If the PTEs were write-protected by uffd-wp in any form, arm special PTEs
|
| 1693 | * to replace none PTEs. NOTE! This should only be called when the PTEs are
|
| 1694 | * already cleared so we will never accidentally replace something valuable.
|
| 1695 | * Meanwhile none PTEs also mean we are not demoting the PTEs so a TLB flush is
|
| 1696 | * not needed. E.g., when the PTEs were cleared, the caller should have taken
|
| 1697 | * care of the TLB flush.
|
| 1698 | *
|
| 1699 | * Must be called with the page table lock held so that no thread will see the
|
| 1700 | * none PTEs, and if they see them, they'll fault and serialize at the page table
|
| 1701 | * lock.
|
| 1702 | *
|
| 1703 | * Returns true if uffd-wp PTEs were installed, false otherwise.
|
| 1704 |
|
| 1705 | bool cond_install_uffd_wp_ptes(struct vm_area_struct *vma,
|
| 1706 | unsigned long addr, pte_t *ptep, pte_t pte,
|
| 1707 | unsigned long nr_ptes)
|
| 1708 | {
|
| 1709 | bool arm_uffd_pte = false;
|
| 1710 |
|
| 1711 | if (!uffd_supports_wp_marker())
|
| 1712 | return false;
|
| 1713 |
|
| 1714 |
|
| 1715 | WARN_ON_ONCE(!pte_none(ptep_get(ptep)));
|
| 1716 |
|
| 1717 | |
| 1718 | * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole
|
| 1719 | * thing, because when zapping either it means it's dropping the
|
| 1720 | * page, or in TTU where the present pte will be quickly replaced
|
| 1721 | * with a swap pte. There's no way of leaking the bit.
|
| 1722 |
|
| 1723 | if (vma_is_anonymous(vma) || !userfaultfd_wp(vma))
|
| 1724 | return false;
|
| 1725 |
|
| 1726 |
|
| 1727 | if (unlikely(pte_present(pte) && pte_uffd(pte)))
|
| 1728 | arm_uffd_pte = true;
|
| 1729 |
|
| 1730 | |
| 1731 | * A uffd-wp wr-protected swap pte. Note: this should even cover an
|
| 1732 | * existing pte marker with uffd-wp bit set.
|
| 1733 |
|
| 1734 | if (unlikely(pte_swp_uffd_any(pte)))
|
| 1735 | arm_uffd_pte = true;
|
| 1736 |
|
| 1737 | if (likely(!arm_uffd_pte))
|
| 1738 | return false;
|
| 1739 |
|
| 1740 | for (;;) {
|
| 1741 | set_pte_at(vma->vm_mm, addr, ptep,
|
| 1742 | make_pte_marker(PTE_MARKER_UFFD_WP));
|
| 1743 | if (--nr_ptes == 0)
|
| 1744 | break;
|
| 1745 | ptep++;
|
| 1746 | addr += PAGE_SIZE;
|
| 1747 | }
|
| 1748 |
|
| 1749 | return true;
|
| 1750 | }
|
| 1751 |
|
| 1752 | |
| 1753 | * This function makes sure that we'll replace the none pte with an uffd-wp
|
| 1754 | * swap special pte marker when necessary. Must be with the pgtable lock held.
|
| 1755 | *
|
| 1756 | * Returns true if uffd-wp ptes was installed, false otherwise.
|
| 1757 |
|
| 1758 | static inline bool
|
| 1759 | zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
|
| 1760 | unsigned long addr, pte_t *pte, int nr,
|
| 1761 | struct zap_details *details, pte_t pteval)
|
| 1762 | {
|
| 1763 | if (zap_drop_markers(details))
|
| 1764 | return false;
|
| 1765 |
|
| 1766 | return cond_install_uffd_wp_ptes(vma, addr, pte, pteval, nr);
|
| 1767 | }
|
| 1768 |
|
| 1769 | static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
|
| 1770 | struct vm_area_struct *vma, struct folio *folio,
|
| 1771 | struct page *page, pte_t *pte, pte_t ptent, unsigned int nr,
|
| 1772 | unsigned long addr, struct zap_details *details, int *rss,
|
| 1773 | bool *force_flush, bool *force_break, bool *any_skipped)
|
| 1774 | {
|
| 1775 | struct mm_struct *mm = tlb->mm;
|
| 1776 | bool delay_rmap = false;
|
| 1777 |
|
| 1778 | if (!folio_test_anon(folio)) {
|
| 1779 | ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
|
| 1780 | if (pte_dirty(ptent)) {
|
| 1781 | folio_mark_dirty(folio);
|
| 1782 | if (tlb_delay_rmap(tlb)) {
|
| 1783 | delay_rmap = true;
|
| 1784 | *force_flush = true;
|
| 1785 | }
|
| 1786 | }
|
| 1787 | if (pte_young(ptent) && likely(vma_has_recency(vma)))
|
| 1788 | folio_mark_accessed(folio);
|
| 1789 | rss[mm_counter(folio)] -= nr;
|
| 1790 | } else {
|
| 1791 |
|
| 1792 | clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
|
| 1793 | rss[MM_ANONPAGES] -= nr;
|
| 1794 | }
|
| 1795 |
|
| 1796 | arch_check_zapped_pte(vma, ptent);
|
| 1797 | tlb_remove_tlb_entries(tlb, pte, nr, addr);
|
| 1798 | if (unlikely(userfaultfd_pte_wp(vma, ptent)))
|
| 1799 | *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte,
|
| 1800 | nr, details, ptent);
|
| 1801 |
|
| 1802 | if (!delay_rmap) {
|
| 1803 | folio_remove_rmap_ptes(folio, page, nr, vma);
|
| 1804 |
|
| 1805 | if (unlikely(folio_mapcount(folio) < 0))
|
| 1806 | print_bad_pte(vma, addr, ptent, page);
|
| 1807 | }
|
| 1808 | if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
|
| 1809 | *force_flush = true;
|
| 1810 | *force_break = true;
|
| 1811 | }
|
| 1812 | }
|
| 1813 |
|
| 1814 | |
| 1815 | * Zap or skip at least one present PTE, trying to batch-process subsequent
|
| 1816 | * PTEs that map consecutive pages of the same folio.
|
| 1817 | *
|
| 1818 | * Returns the number of processed (skipped or zapped) PTEs (at least 1).
|
| 1819 |
|
| 1820 | static inline int zap_present_ptes(struct mmu_gather *tlb,
|
| 1821 | struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
|
| 1822 | unsigned int max_nr, unsigned long addr,
|
| 1823 | struct zap_details *details, int *rss, bool *force_flush,
|
| 1824 | bool *force_break, bool *any_skipped)
|
| 1825 | {
|
| 1826 | struct mm_struct *mm = tlb->mm;
|
| 1827 | struct folio *folio;
|
| 1828 | struct page *page;
|
| 1829 | int nr;
|
| 1830 |
|
| 1831 | page = vm_normal_page(vma, addr, ptent);
|
| 1832 | if (!page) {
|
| 1833 |
|
| 1834 | ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);
|
| 1835 | arch_check_zapped_pte(vma, ptent);
|
| 1836 | tlb_remove_tlb_entry(tlb, pte, addr);
|
| 1837 | if (userfaultfd_pte_wp(vma, ptent))
|
| 1838 | *any_skipped = zap_install_uffd_wp_if_needed(vma, addr,
|
| 1839 | pte, 1, details, ptent);
|
| 1840 | ksm_might_unmap_zero_page(mm, ptent);
|
| 1841 | return 1;
|
| 1842 | }
|
| 1843 |
|
| 1844 | folio = page_folio(page);
|
| 1845 | if (unlikely(!should_zap_folio(details, folio))) {
|
| 1846 | *any_skipped = true;
|
| 1847 | return 1;
|
| 1848 | }
|
| 1849 |
|
| 1850 | |
| 1851 | * Make sure that the common "small folio" case is as fast as possible
|
| 1852 | * by keeping the batching logic separate.
|
| 1853 |
|
| 1854 | if (unlikely(folio_test_large(folio) && max_nr != 1)) {
|
| 1855 | nr = folio_pte_batch(folio, pte, ptent, max_nr);
|
| 1856 | zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,
|
| 1857 | addr, details, rss, force_flush,
|
| 1858 | force_break, any_skipped);
|
| 1859 | return nr;
|
| 1860 | }
|
| 1861 | zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr,
|
| 1862 | details, rss, force_flush, force_break, any_skipped);
|
| 1863 | return 1;
|
| 1864 | }
|
| 1865 |
|
| 1866 | static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,
|
| 1867 | struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
|
| 1868 | unsigned int max_nr, unsigned long addr,
|
| 1869 | struct zap_details *details, int *rss, bool *any_skipped)
|
| 1870 | {
|
| 1871 | softleaf_t entry;
|
| 1872 | int nr = 1;
|
| 1873 |
|
| 1874 | *any_skipped = true;
|
| 1875 | entry = softleaf_from_pte(ptent);
|
| 1876 | if (softleaf_is_device_private(entry) ||
|
| 1877 | softleaf_is_device_exclusive(entry)) {
|
| 1878 | struct page *page = softleaf_to_page(entry);
|
| 1879 | struct folio *folio = page_folio(page);
|
| 1880 |
|
| 1881 | if (unlikely(!should_zap_folio(details, folio)))
|
| 1882 | return 1;
|
| 1883 | |
| 1884 | * Both device private/exclusive mappings should only
|
| 1885 | * work with anonymous page so far, so we don't need to
|
| 1886 | * consider uffd-wp bit when zap. For more information,
|
| 1887 | * see zap_install_uffd_wp_if_needed().
|
| 1888 |
|
| 1889 | WARN_ON_ONCE(!folio_test_anon(folio));
|
| 1890 | rss[mm_counter(folio)]--;
|
| 1891 | folio_remove_rmap_pte(folio, page, vma);
|
| 1892 | folio_put(folio);
|
| 1893 | } else if (softleaf_is_swap(entry)) {
|
| 1894 |
|
| 1895 | if (!should_zap_cows(details))
|
| 1896 | return 1;
|
| 1897 |
|
| 1898 | nr = swap_pte_batch(pte, max_nr, ptent);
|
| 1899 | rss[MM_SWAPENTS] -= nr;
|
| 1900 | swap_put_entries_direct(entry, nr);
|
| 1901 | } else if (softleaf_is_migration(entry)) {
|
| 1902 | struct folio *folio = softleaf_to_folio(entry);
|
| 1903 |
|
| 1904 | if (!should_zap_folio(details, folio))
|
| 1905 | return 1;
|
| 1906 | rss[mm_counter(folio)]--;
|
| 1907 | } else if (softleaf_is_uffd_wp_marker(entry)) {
|
| 1908 | |
| 1909 | * For anon: always drop the marker; for file: only
|
| 1910 | * drop the marker if explicitly requested.
|
| 1911 |
|
| 1912 | if (!vma_is_anonymous(vma) && !zap_drop_markers(details))
|
| 1913 | return 1;
|
| 1914 | } else if (softleaf_is_guard_marker(entry)) {
|
| 1915 | |
| 1916 | * Ordinary zapping should not remove guard PTE
|
| 1917 | * markers. Only do so if we should remove PTE markers
|
| 1918 | * in general.
|
| 1919 |
|
| 1920 | if (!zap_drop_markers(details))
|
| 1921 | return 1;
|
| 1922 | } else if (softleaf_is_hwpoison(entry) ||
|
| 1923 | softleaf_is_poison_marker(entry)) {
|
| 1924 | if (!should_zap_cows(details))
|
| 1925 | return 1;
|
| 1926 | } else {
|
| 1927 |
|
| 1928 | pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
|
| 1929 | WARN_ON_ONCE(1);
|
| 1930 | }
|
| 1931 | clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr);
|
| 1932 | *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);
|
| 1933 |
|
| 1934 | return nr;
|
| 1935 | }
|
| 1936 |
|
| 1937 | static inline int do_zap_pte_range(struct mmu_gather *tlb,
|
| 1938 | struct vm_area_struct *vma, pte_t *pte,
|
| 1939 | unsigned long addr, unsigned long end,
|
| 1940 | struct zap_details *details, int *rss,
|
| 1941 | bool *force_flush, bool *force_break,
|
| 1942 | bool *any_skipped)
|
| 1943 | {
|
| 1944 | pte_t ptent = ptep_get(pte);
|
| 1945 | int max_nr = (end - addr) / PAGE_SIZE;
|
| 1946 | int nr = 0;
|
| 1947 |
|
| 1948 |
|
| 1949 | if (pte_none(ptent)) {
|
| 1950 | for (nr = 1; nr < max_nr; nr++) {
|
| 1951 | ptent = ptep_get(pte + nr);
|
| 1952 | if (!pte_none(ptent))
|
| 1953 | break;
|
| 1954 | }
|
| 1955 | max_nr -= nr;
|
| 1956 | if (!max_nr)
|
| 1957 | return nr;
|
| 1958 | pte += nr;
|
| 1959 | addr += nr * PAGE_SIZE;
|
| 1960 | }
|
| 1961 |
|
| 1962 | if (pte_present(ptent))
|
| 1963 | nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr,
|
| 1964 | details, rss, force_flush, force_break,
|
| 1965 | any_skipped);
|
| 1966 | else
|
| 1967 | nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr,
|
| 1968 | details, rss, any_skipped);
|
| 1969 |
|
| 1970 | return nr;
|
| 1971 | }
|
| 1972 |
|
| 1973 | static bool pte_table_reclaim_possible(unsigned long start, unsigned long end,
|
| 1974 | struct zap_details *details)
|
| 1975 | {
|
| 1976 | if (!IS_ENABLED(CONFIG_PT_RECLAIM))
|
| 1977 | return false;
|
| 1978 |
|
| 1979 | return details && details->reclaim_pt && (end - start >= PMD_SIZE);
|
| 1980 | }
|
| 1981 |
|
| 1982 | static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd,
|
| 1983 | spinlock_t *ptl, pmd_t *pmdval)
|
| 1984 | {
|
| 1985 | spinlock_t *pml = pmd_lockptr(mm, pmd);
|
| 1986 |
|
| 1987 | if (ptl != pml && !spin_trylock(pml))
|
| 1988 | return false;
|
| 1989 |
|
| 1990 | *pmdval = pmdp_get(pmd);
|
| 1991 | pmd_clear(pmd);
|
| 1992 | if (ptl != pml)
|
| 1993 | spin_unlock(pml);
|
| 1994 | return true;
|
| 1995 | }
|
| 1996 |
|
| 1997 | static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd,
|
| 1998 | unsigned long addr, pmd_t *pmdval)
|
| 1999 | {
|
| 2000 | spinlock_t *pml, *ptl = NULL;
|
| 2001 | pte_t *start_pte, *pte;
|
| 2002 | int i;
|
| 2003 |
|
| 2004 | pml = pmd_lock(mm, pmd);
|
| 2005 | start_pte = pte_offset_map_rw_nolock(mm, pmd, addr, pmdval, &ptl);
|
| 2006 | if (!start_pte)
|
| 2007 | goto out_ptl;
|
| 2008 | if (ptl != pml)
|
| 2009 | spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);
|
| 2010 |
|
| 2011 | for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) {
|
| 2012 | if (!pte_none(ptep_get(pte)))
|
| 2013 | goto out_ptl;
|
| 2014 | }
|
| 2015 | pte_unmap(start_pte);
|
| 2016 |
|
| 2017 | pmd_clear(pmd);
|
| 2018 |
|
| 2019 | if (ptl != pml)
|
| 2020 | spin_unlock(ptl);
|
| 2021 | spin_unlock(pml);
|
| 2022 | return true;
|
| 2023 | out_ptl:
|
| 2024 | if (start_pte)
|
| 2025 | pte_unmap_unlock(start_pte, ptl);
|
| 2026 | if (ptl != pml)
|
| 2027 | spin_unlock(pml);
|
| 2028 | return false;
|
| 2029 | }
|
| 2030 |
|
| 2031 | static unsigned long zap_pte_range(struct mmu_gather *tlb,
|
| 2032 | struct vm_area_struct *vma, pmd_t *pmd,
|
| 2033 | unsigned long addr, unsigned long end,
|
| 2034 | struct zap_details *details)
|
| 2035 | {
|
| 2036 | bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details);
|
| 2037 | bool force_flush = false, force_break = false;
|
| 2038 | struct mm_struct *mm = tlb->mm;
|
| 2039 | int rss[NR_MM_COUNTERS];
|
| 2040 | spinlock_t *ptl;
|
| 2041 | pte_t *start_pte;
|
| 2042 | pte_t *pte;
|
| 2043 | pmd_t pmdval;
|
| 2044 | unsigned long start = addr;
|
| 2045 | bool direct_reclaim = true;
|
| 2046 | int nr;
|
| 2047 |
|
| 2048 | retry:
|
| 2049 | tlb_change_page_size(tlb, PAGE_SIZE);
|
| 2050 | init_rss_vec(rss);
|
| 2051 | start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
|
| 2052 | if (!pte)
|
| 2053 | return addr;
|
| 2054 |
|
| 2055 | flush_tlb_batched_pending(mm);
|
| 2056 | lazy_mmu_mode_enable();
|
| 2057 | do {
|
| 2058 | bool any_skipped = false;
|
| 2059 |
|
| 2060 | if (need_resched()) {
|
| 2061 | direct_reclaim = false;
|
| 2062 | break;
|
| 2063 | }
|
| 2064 |
|
| 2065 | nr = do_zap_pte_range(tlb, vma, pte, addr, end, details, rss,
|
| 2066 | &force_flush, &force_break, &any_skipped);
|
| 2067 | if (any_skipped)
|
| 2068 | can_reclaim_pt = false;
|
| 2069 | if (unlikely(force_break)) {
|
| 2070 | addr += nr * PAGE_SIZE;
|
| 2071 | direct_reclaim = false;
|
| 2072 | break;
|
| 2073 | }
|
| 2074 | } while (pte += nr, addr += PAGE_SIZE * nr, addr != end);
|
| 2075 |
|
| 2076 | |
| 2077 | * Fast path: try to hold the pmd lock and unmap the PTE page.
|
| 2078 | *
|
| 2079 | * If the pte lock was released midway (retry case), or if the attempt
|
| 2080 | * to hold the pmd lock failed, then we need to recheck all pte entries
|
| 2081 | * to ensure they are still none, thereby preventing the pte entries
|
| 2082 | * from being repopulated by another thread.
|
| 2083 |
|
| 2084 | if (can_reclaim_pt && direct_reclaim && addr == end)
|
| 2085 | direct_reclaim = zap_empty_pte_table(mm, pmd, ptl, &pmdval);
|
| 2086 |
|
| 2087 | add_mm_rss_vec(mm, rss);
|
| 2088 | lazy_mmu_mode_disable();
|
| 2089 |
|
| 2090 |
|
| 2091 | if (force_flush) {
|
| 2092 | tlb_flush_mmu_tlbonly(tlb);
|
| 2093 | tlb_flush_rmaps(tlb, vma);
|
| 2094 | }
|
| 2095 | pte_unmap_unlock(start_pte, ptl);
|
| 2096 |
|
| 2097 | |
| 2098 | * If we forced a TLB flush (either due to running out of
|
| 2099 | * batch buffers or because we needed to flush dirty TLB
|
| 2100 | * entries before releasing the ptl), free the batched
|
| 2101 | * memory too. Come back again if we didn't do everything.
|
| 2102 |
|
| 2103 | if (force_flush)
|
| 2104 | tlb_flush_mmu(tlb);
|
| 2105 |
|
| 2106 | if (addr != end) {
|
| 2107 | cond_resched();
|
| 2108 | force_flush = false;
|
| 2109 | force_break = false;
|
| 2110 | goto retry;
|
| 2111 | }
|
| 2112 |
|
| 2113 | if (can_reclaim_pt) {
|
| 2114 | if (direct_reclaim || zap_pte_table_if_empty(mm, pmd, start, &pmdval)) {
|
| 2115 | pte_free_tlb(tlb, pmd_pgtable(pmdval), start);
|
| 2116 | mm_dec_nr_ptes(mm);
|
| 2117 | }
|
| 2118 | }
|
| 2119 |
|
| 2120 | return addr;
|
| 2121 | }
|
| 2122 |
|
| 2123 | static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
|
| 2124 | struct vm_area_struct *vma, pud_t *pud,
|
| 2125 | unsigned long addr, unsigned long end,
|
| 2126 | struct zap_details *details)
|
| 2127 | {
|
| 2128 | pmd_t *pmd;
|
| 2129 | unsigned long next;
|
| 2130 |
|
| 2131 | pmd = pmd_offset(pud, addr);
|
| 2132 | do {
|
| 2133 | next = pmd_addr_end(addr, end);
|
| 2134 | if (pmd_is_huge(*pmd)) {
|
| 2135 | if (next - addr != HPAGE_PMD_SIZE)
|
| 2136 | __split_huge_pmd(vma, pmd, addr, false);
|
| 2137 | else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
|
| 2138 | addr = next;
|
| 2139 | continue;
|
| 2140 | }
|
| 2141 |
|
| 2142 | } else if (details && details->single_folio &&
|
| 2143 | folio_test_pmd_mappable(details->single_folio) &&
|
| 2144 | next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
|
| 2145 | sync_with_folio_pmd_zap(tlb->mm, pmd);
|
| 2146 | }
|
| 2147 | if (pmd_none(*pmd)) {
|
| 2148 | addr = next;
|
| 2149 | continue;
|
| 2150 | }
|
| 2151 | addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
|
| 2152 | if (addr != next)
|
| 2153 | pmd--;
|
| 2154 | } while (pmd++, cond_resched(), addr != end);
|
| 2155 |
|
| 2156 | return addr;
|
| 2157 | }
|
| 2158 |
|
| 2159 | static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
|
| 2160 | struct vm_area_struct *vma, p4d_t *p4d,
|
| 2161 | unsigned long addr, unsigned long end,
|
| 2162 | struct zap_details *details)
|
| 2163 | {
|
| 2164 | pud_t *pud;
|
| 2165 | unsigned long next;
|
| 2166 |
|
| 2167 | pud = pud_offset(p4d, addr);
|
| 2168 | do {
|
| 2169 | next = pud_addr_end(addr, end);
|
| 2170 | if (pud_trans_huge(*pud)) {
|
| 2171 | if (next - addr != HPAGE_PUD_SIZE)
|
| 2172 | split_huge_pud(vma, pud, addr);
|
| 2173 | else if (zap_huge_pud(tlb, vma, pud, addr))
|
| 2174 | goto next;
|
| 2175 |
|
| 2176 | }
|
| 2177 | if (pud_none_or_clear_bad(pud))
|
| 2178 | continue;
|
| 2179 | next = zap_pmd_range(tlb, vma, pud, addr, next, details);
|
| 2180 | next:
|
| 2181 | cond_resched();
|
| 2182 | } while (pud++, addr = next, addr != end);
|
| 2183 |
|
| 2184 | return addr;
|
| 2185 | }
|
| 2186 |
|
| 2187 | static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
|
| 2188 | struct vm_area_struct *vma, pgd_t *pgd,
|
| 2189 | unsigned long addr, unsigned long end,
|
| 2190 | struct zap_details *details)
|
| 2191 | {
|
| 2192 | p4d_t *p4d;
|
| 2193 | unsigned long next;
|
| 2194 |
|
| 2195 | p4d = p4d_offset(pgd, addr);
|
| 2196 | do {
|
| 2197 | next = p4d_addr_end(addr, end);
|
| 2198 | if (p4d_none_or_clear_bad(p4d))
|
| 2199 | continue;
|
| 2200 | next = zap_pud_range(tlb, vma, p4d, addr, next, details);
|
| 2201 | } while (p4d++, addr = next, addr != end);
|
| 2202 |
|
| 2203 | return addr;
|
| 2204 | }
|
| 2205 |
|
| 2206 | static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
|
| 2207 | unsigned long start, unsigned long end,
|
| 2208 | struct zap_details *details)
|
| 2209 | {
|
| 2210 | const bool reaping = details && details->reaping;
|
| 2211 |
|
| 2212 | VM_WARN_ON_ONCE(start >= end || !range_in_vma(vma, start, end));
|
| 2213 |
|
| 2214 |
|
| 2215 | if (vma->vm_file && !reaping)
|
| 2216 | uprobe_munmap(vma, start, end);
|
| 2217 |
|
| 2218 | if (unlikely(is_vm_hugetlb_page(vma))) {
|
| 2219 | zap_flags_t zap_flags = details ? details->zap_flags : 0;
|
| 2220 |
|
| 2221 | VM_WARN_ON_ONCE(reaping);
|
| 2222 | |
| 2223 | * vm_file will be NULL when we fail early while instantiating
|
| 2224 | * a new mapping. In this case, no pages were mapped yet and
|
| 2225 | * there is nothing to do.
|
| 2226 |
|
| 2227 | if (!vma->vm_file)
|
| 2228 | return;
|
| 2229 | __unmap_hugepage_range(tlb, vma, start, end, NULL, zap_flags);
|
| 2230 | } else {
|
| 2231 | unsigned long next, addr = start;
|
| 2232 | pgd_t *pgd;
|
| 2233 |
|
| 2234 | tlb_start_vma(tlb, vma);
|
| 2235 | pgd = pgd_offset(vma->vm_mm, addr);
|
| 2236 | do {
|
| 2237 | next = pgd_addr_end(addr, end);
|
| 2238 | if (pgd_none_or_clear_bad(pgd))
|
| 2239 | continue;
|
| 2240 | next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
|
| 2241 | } while (pgd++, addr = next, addr != end);
|
| 2242 | tlb_end_vma(tlb, vma);
|
| 2243 | }
|
| 2244 | }
|
| 2245 |
|
| 2246 | |
| 2247 | * zap_vma_for_reaping - zap all page table entries in the vma without blocking
|
| 2248 | * @vma: The vma to zap.
|
| 2249 | *
|
| 2250 | * Zap all page table entries in the vma without blocking for use by the oom
|
| 2251 | * killer. Hugetlb vmas are not supported.
|
| 2252 | *
|
| 2253 | * Returns: 0 on success, -EBUSY if we would have to block.
|
| 2254 |
|
| 2255 | int zap_vma_for_reaping(struct vm_area_struct *vma)
|
| 2256 | {
|
| 2257 | struct zap_details details = {
|
| 2258 | .reaping = true,
|
| 2259 | };
|
| 2260 | struct mmu_notifier_range range;
|
| 2261 | struct mmu_gather tlb;
|
| 2262 |
|
| 2263 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
| 2264 | vma->vm_start, vma->vm_end);
|
| 2265 | tlb_gather_mmu(&tlb, vma->vm_mm);
|
| 2266 | if (mmu_notifier_invalidate_range_start_nonblock(&range)) {
|
| 2267 | tlb_finish_mmu(&tlb);
|
| 2268 | return -EBUSY;
|
| 2269 | }
|
| 2270 | __zap_vma_range(&tlb, vma, range.start, range.end, &details);
|
| 2271 | mmu_notifier_invalidate_range_end(&range);
|
| 2272 | tlb_finish_mmu(&tlb);
|
| 2273 | return 0;
|
| 2274 | }
|
| 2275 |
|
| 2276 | |
| 2277 | * unmap_vmas - unmap a range of memory covered by a list of vma's
|
| 2278 | * @tlb: address of the caller's struct mmu_gather
|
| 2279 | * @unmap: The unmap_desc
|
| 2280 | *
|
| 2281 | * Unmap all pages in the vma list.
|
| 2282 | *
|
| 2283 | * Only addresses between `start' and `end' will be unmapped.
|
| 2284 | *
|
| 2285 | * The VMA list must be sorted in ascending virtual address order.
|
| 2286 | *
|
| 2287 | * unmap_vmas() assumes that the caller will flush the whole unmapped address
|
| 2288 | * range after unmap_vmas() returns. So the only responsibility here is to
|
| 2289 | * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
|
| 2290 | * drops the lock and schedules.
|
| 2291 |
|
| 2292 | void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap)
|
| 2293 | {
|
| 2294 | struct vm_area_struct *vma;
|
| 2295 | struct mmu_notifier_range range;
|
| 2296 | struct zap_details details = {
|
| 2297 | .zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
|
| 2298 | };
|
| 2299 |
|
| 2300 | vma = unmap->first;
|
| 2301 | mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
|
| 2302 | unmap->vma_start, unmap->vma_end);
|
| 2303 | mmu_notifier_invalidate_range_start(&range);
|
| 2304 | do {
|
| 2305 | unsigned long start = max(vma->vm_start, unmap->vma_start);
|
| 2306 | unsigned long end = min(vma->vm_end, unmap->vma_end);
|
| 2307 |
|
| 2308 | hugetlb_zap_begin(vma, &start, &end);
|
| 2309 | __zap_vma_range(tlb, vma, start, end, &details);
|
| 2310 | hugetlb_zap_end(vma, &details);
|
| 2311 | vma = mas_find(unmap->mas, unmap->tree_end - 1);
|
| 2312 | } while (vma);
|
| 2313 | mmu_notifier_invalidate_range_end(&range);
|
| 2314 | }
|
| 2315 |
|
| 2316 | |
| 2317 | * zap_vma_range_batched - zap page table entries in a vma range
|
| 2318 | * @tlb: pointer to the caller's struct mmu_gather
|
| 2319 | * @vma: the vma covering the range to zap
|
| 2320 | * @address: starting address of the range to zap
|
| 2321 | * @size: number of bytes to zap
|
| 2322 | * @details: details specifying zapping behavior
|
| 2323 | *
|
| 2324 | * @tlb must not be NULL. The provided address range must be fully
|
| 2325 | * contained within @vma. If @vma is for hugetlb, @tlb is flushed and
|
| 2326 | * re-initialized by this function.
|
| 2327 | *
|
| 2328 | * If @details is NULL, this function will zap all page table entries.
|
| 2329 |
|
| 2330 | void zap_vma_range_batched(struct mmu_gather *tlb,
|
| 2331 | struct vm_area_struct *vma, unsigned long address,
|
| 2332 | unsigned long size, struct zap_details *details)
|
| 2333 | {
|
| 2334 | const unsigned long end = address + size;
|
| 2335 | struct mmu_notifier_range range;
|
| 2336 |
|
| 2337 | VM_WARN_ON_ONCE(!tlb || tlb->mm != vma->vm_mm);
|
| 2338 |
|
| 2339 | if (unlikely(!size))
|
| 2340 | return;
|
| 2341 |
|
| 2342 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
| 2343 | address, end);
|
| 2344 | hugetlb_zap_begin(vma, &range.start, &range.end);
|
| 2345 | update_hiwater_rss(vma->vm_mm);
|
| 2346 | mmu_notifier_invalidate_range_start(&range);
|
| 2347 | |
| 2348 | * unmap 'address-end' not 'range.start-range.end' as range
|
| 2349 | * could have been expanded for hugetlb pmd sharing.
|
| 2350 |
|
| 2351 | __zap_vma_range(tlb, vma, address, end, details);
|
| 2352 | mmu_notifier_invalidate_range_end(&range);
|
| 2353 | if (is_vm_hugetlb_page(vma)) {
|
| 2354 | |
| 2355 | * flush tlb and free resources before hugetlb_zap_end(), to
|
| 2356 | * avoid concurrent page faults' allocation failure.
|
| 2357 |
|
| 2358 | tlb_finish_mmu(tlb);
|
| 2359 | hugetlb_zap_end(vma, details);
|
| 2360 | tlb_gather_mmu(tlb, vma->vm_mm);
|
| 2361 | }
|
| 2362 | }
|
| 2363 |
|
| 2364 | |
| 2365 | * zap_vma_range - zap all page table entries in a vma range
|
| 2366 | * @vma: the vma covering the range to zap
|
| 2367 | * @address: starting address of the range to zap
|
| 2368 | * @size: number of bytes to zap
|
| 2369 | *
|
| 2370 | * The provided address range must be fully contained within @vma.
|
| 2371 |
|
| 2372 | void zap_vma_range(struct vm_area_struct *vma, unsigned long address,
|
| 2373 | unsigned long size)
|
| 2374 | {
|
| 2375 | struct mmu_gather tlb;
|
| 2376 |
|
| 2377 | tlb_gather_mmu(&tlb, vma->vm_mm);
|
| 2378 | zap_vma_range_batched(&tlb, vma, address, size, NULL);
|
| 2379 | tlb_finish_mmu(&tlb);
|
| 2380 | }
|
| 2381 |
|
| 2382 | |
| 2383 | * zap_special_vma_range - zap all page table entries in a special vma range
|
| 2384 | * @vma: the vma covering the range to zap
|
| 2385 | * @address: starting address of the range to zap
|
| 2386 | * @size: number of bytes to zap
|
| 2387 | *
|
| 2388 | * This function does nothing when the provided address range is not fully
|
| 2389 | * contained in @vma, or when the @vma is not VM_PFNMAP or VM_MIXEDMAP.
|
| 2390 |
|
| 2391 | void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address,
|
| 2392 | unsigned long size)
|
| 2393 | {
|
| 2394 | if (!range_in_vma(vma, address, address + size) ||
|
| 2395 | !(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)))
|
| 2396 | return;
|
| 2397 |
|
| 2398 | zap_vma_range(vma, address, size);
|
| 2399 | }
|
| 2400 | EXPORT_SYMBOL_GPL(zap_special_vma_range);
|
| 2401 |
|
| 2402 | static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
|
| 2403 | {
|
| 2404 | pgd_t *pgd;
|
| 2405 | p4d_t *p4d;
|
| 2406 | pud_t *pud;
|
| 2407 | pmd_t *pmd;
|
| 2408 |
|
| 2409 | pgd = pgd_offset(mm, addr);
|
| 2410 | p4d = p4d_alloc(mm, pgd, addr);
|
| 2411 | if (!p4d)
|
| 2412 | return NULL;
|
| 2413 | pud = pud_alloc(mm, p4d, addr);
|
| 2414 | if (!pud)
|
| 2415 | return NULL;
|
| 2416 | pmd = pmd_alloc(mm, pud, addr);
|
| 2417 | if (!pmd)
|
| 2418 | return NULL;
|
| 2419 |
|
| 2420 | VM_BUG_ON(pmd_trans_huge(*pmd));
|
| 2421 | return pmd;
|
| 2422 | }
|
| 2423 |
|
| 2424 | pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
|
| 2425 | spinlock_t **ptl)
|
| 2426 | {
|
| 2427 | pmd_t *pmd = walk_to_pmd(mm, addr);
|
| 2428 |
|
| 2429 | if (!pmd)
|
| 2430 | return NULL;
|
| 2431 | return pte_alloc_map_lock(mm, pmd, addr, ptl);
|
| 2432 | }
|
| 2433 |
|
| 2434 | static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma)
|
| 2435 | {
|
| 2436 | VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP);
|
| 2437 | |
| 2438 | * Whoever wants to forbid the zeropage after some zeropages
|
| 2439 | * might already have been mapped has to scan the page tables and
|
| 2440 | * bail out on any zeropages. Zeropages in COW mappings can
|
| 2441 | * be unshared using FAULT_FLAG_UNSHARE faults.
|
| 2442 |
|
| 2443 | if (mm_forbids_zeropage(vma->vm_mm))
|
| 2444 | return false;
|
| 2445 |
|
| 2446 | if (vma_is_cow_mapping(vma))
|
| 2447 | return true;
|
| 2448 |
|
| 2449 | if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)))
|
| 2450 | return true;
|
| 2451 | |
| 2452 | * Why not allow any VMA that has vm_ops->pfn_mkwrite? GUP could
|
| 2453 | * find the shared zeropage and longterm-pin it, which would
|
| 2454 | * be problematic as soon as the zeropage gets replaced by a different
|
| 2455 | * page due to vma->vm_ops->pfn_mkwrite, because what's mapped would
|
| 2456 | * now differ to what GUP looked up. FSDAX is incompatible to
|
| 2457 | * FOLL_LONGTERM and VM_IO is incompatible to GUP completely (see
|
| 2458 | * check_vma_flags).
|
| 2459 |
|
| 2460 | return vma->vm_ops && vma->vm_ops->pfn_mkwrite &&
|
| 2461 | (vma_is_fsdax(vma) || vma->vm_flags & VM_IO);
|
| 2462 | }
|
| 2463 |
|
| 2464 | static int validate_page_before_insert(struct vm_area_struct *vma,
|
| 2465 | struct page *page)
|
| 2466 | {
|
| 2467 | struct folio *folio = page_folio(page);
|
| 2468 |
|
| 2469 | if (!folio_ref_count(folio))
|
| 2470 | return -EINVAL;
|
| 2471 | if (unlikely(is_zero_folio(folio))) {
|
| 2472 | if (!vm_mixed_zeropage_allowed(vma))
|
| 2473 | return -EINVAL;
|
| 2474 | return 0;
|
| 2475 | }
|
| 2476 | if (folio_test_anon(folio) || page_has_type(page))
|
| 2477 | return -EINVAL;
|
| 2478 | flush_dcache_folio(folio);
|
| 2479 | return 0;
|
| 2480 | }
|
| 2481 |
|
| 2482 | static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
|
| 2483 | unsigned long addr, struct page *page,
|
| 2484 | pgprot_t prot, bool mkwrite)
|
| 2485 | {
|
| 2486 | struct folio *folio = page_folio(page);
|
| 2487 | pte_t pteval = ptep_get(pte);
|
| 2488 |
|
| 2489 | if (!pte_none(pteval)) {
|
| 2490 | if (!mkwrite)
|
| 2491 | return -EBUSY;
|
| 2492 |
|
| 2493 |
|
| 2494 | if (pte_pfn(pteval) != page_to_pfn(page)) {
|
| 2495 | WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval)));
|
| 2496 | return -EFAULT;
|
| 2497 | }
|
| 2498 | pteval = maybe_mkwrite(pteval, vma);
|
| 2499 | pteval = pte_mkyoung(pteval);
|
| 2500 | if (ptep_set_access_flags(vma, addr, pte, pteval, 1))
|
| 2501 | update_mmu_cache(vma, addr, pte);
|
| 2502 | return 0;
|
| 2503 | }
|
| 2504 |
|
| 2505 |
|
| 2506 | pteval = mk_pte(page, prot);
|
| 2507 | if (unlikely(is_zero_folio(folio))) {
|
| 2508 | pteval = pte_mkspecial(pteval);
|
| 2509 | } else {
|
| 2510 | folio_get(folio);
|
| 2511 | pteval = mk_pte(page, prot);
|
| 2512 | if (mkwrite) {
|
| 2513 | pteval = pte_mkyoung(pteval);
|
| 2514 | pteval = maybe_mkwrite(pte_mkdirty(pteval), vma);
|
| 2515 | }
|
| 2516 | inc_mm_counter(vma->vm_mm, mm_counter_file(folio));
|
| 2517 | folio_add_file_rmap_pte(folio, page, vma);
|
| 2518 | }
|
| 2519 | set_pte_at(vma->vm_mm, addr, pte, pteval);
|
| 2520 | return 0;
|
| 2521 | }
|
| 2522 |
|
| 2523 | static int insert_page(struct vm_area_struct *vma, unsigned long addr,
|
| 2524 | struct page *page, pgprot_t prot, bool mkwrite)
|
| 2525 | {
|
| 2526 | int retval;
|
| 2527 | pte_t *pte;
|
| 2528 | spinlock_t *ptl;
|
| 2529 |
|
| 2530 | retval = validate_page_before_insert(vma, page);
|
| 2531 | if (retval)
|
| 2532 | goto out;
|
| 2533 | retval = -ENOMEM;
|
| 2534 | pte = get_locked_pte(vma->vm_mm, addr, &ptl);
|
| 2535 | if (!pte)
|
| 2536 | goto out;
|
| 2537 | retval = insert_page_into_pte_locked(vma, pte, addr, page, prot,
|
| 2538 | mkwrite);
|
| 2539 | pte_unmap_unlock(pte, ptl);
|
| 2540 | out:
|
| 2541 | return retval;
|
| 2542 | }
|
| 2543 |
|
| 2544 | static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
|
| 2545 | unsigned long addr, struct page *page, pgprot_t prot)
|
| 2546 | {
|
| 2547 | int err;
|
| 2548 |
|
| 2549 | err = validate_page_before_insert(vma, page);
|
| 2550 | if (err)
|
| 2551 | return err;
|
| 2552 | return insert_page_into_pte_locked(vma, pte, addr, page, prot, false);
|
| 2553 | }
|
| 2554 |
|
| 2555 | |
| 2556 | * when inserting pages in a loop.
|
| 2557 |
|
| 2558 | static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
|
| 2559 | struct page **pages, unsigned long *num, pgprot_t prot)
|
| 2560 | {
|
| 2561 | pmd_t *pmd = NULL;
|
| 2562 | pte_t *start_pte, *pte;
|
| 2563 | spinlock_t *pte_lock;
|
| 2564 | struct mm_struct *const mm = vma->vm_mm;
|
| 2565 | unsigned long curr_page_idx = 0;
|
| 2566 | unsigned long remaining_pages_total = *num;
|
| 2567 | unsigned long pages_to_write_in_pmd;
|
| 2568 | int ret;
|
| 2569 | more:
|
| 2570 | ret = -EFAULT;
|
| 2571 | pmd = walk_to_pmd(mm, addr);
|
| 2572 | if (!pmd)
|
| 2573 | goto out;
|
| 2574 |
|
| 2575 | pages_to_write_in_pmd = min_t(unsigned long,
|
| 2576 | remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
|
| 2577 |
|
| 2578 |
|
| 2579 | ret = -ENOMEM;
|
| 2580 | if (pte_alloc(mm, pmd))
|
| 2581 | goto out;
|
| 2582 |
|
| 2583 | while (pages_to_write_in_pmd) {
|
| 2584 | int pte_idx = 0;
|
| 2585 | const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
|
| 2586 |
|
| 2587 | start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
|
| 2588 | if (!start_pte) {
|
| 2589 | ret = -EFAULT;
|
| 2590 | goto out;
|
| 2591 | }
|
| 2592 | for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
|
| 2593 | int err = insert_page_in_batch_locked(vma, pte,
|
| 2594 | addr, pages[curr_page_idx], prot);
|
| 2595 | if (unlikely(err)) {
|
| 2596 | pte_unmap_unlock(start_pte, pte_lock);
|
| 2597 | ret = err;
|
| 2598 | remaining_pages_total -= pte_idx;
|
| 2599 | goto out;
|
| 2600 | }
|
| 2601 | addr += PAGE_SIZE;
|
| 2602 | ++curr_page_idx;
|
| 2603 | }
|
| 2604 | pte_unmap_unlock(start_pte, pte_lock);
|
| 2605 | pages_to_write_in_pmd -= batch_size;
|
| 2606 | remaining_pages_total -= batch_size;
|
| 2607 | }
|
| 2608 | if (remaining_pages_total)
|
| 2609 | goto more;
|
| 2610 | ret = 0;
|
| 2611 | out:
|
| 2612 | *num = remaining_pages_total;
|
| 2613 | return ret;
|
| 2614 | }
|
| 2615 |
|
| 2616 | |
| 2617 | * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
|
| 2618 | * @vma: user vma to map to
|
| 2619 | * @addr: target start user address of these pages
|
| 2620 | * @pages: source kernel pages
|
| 2621 | * @num: in: number of pages to map. out: number of pages that were *not*
|
| 2622 | * mapped. (0 means all pages were successfully mapped).
|
| 2623 | *
|
| 2624 | * Preferred over vm_insert_page() when inserting multiple pages.
|
| 2625 | *
|
| 2626 | * In case of error, we may have mapped a subset of the provided
|
| 2627 | * pages. It is the caller's responsibility to account for this case.
|
| 2628 | *
|
| 2629 | * The same restrictions apply as in vm_insert_page().
|
| 2630 |
|
| 2631 | int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
|
| 2632 | struct page **pages, unsigned long *num)
|
| 2633 | {
|
| 2634 | const unsigned long nr_pages = *num;
|
| 2635 | const unsigned long end = addr + PAGE_SIZE * nr_pages;
|
| 2636 |
|
| 2637 | if (!range_in_vma(vma, addr, end))
|
| 2638 | return -EFAULT;
|
| 2639 | if (!(vma->vm_flags & VM_MIXEDMAP)) {
|
| 2640 | VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm));
|
| 2641 | VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP);
|
| 2642 | vm_flags_set(vma, VM_MIXEDMAP);
|
| 2643 | }
|
| 2644 |
|
| 2645 | return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
|
| 2646 | }
|
| 2647 | EXPORT_SYMBOL(vm_insert_pages);
|
| 2648 |
|
| 2649 | int map_kernel_pages_prepare(struct vm_area_desc *desc)
|
| 2650 | {
|
| 2651 | const struct mmap_action *action = &desc->action;
|
| 2652 | const unsigned long addr = action->map_kernel.start;
|
| 2653 | unsigned long nr_pages, end;
|
| 2654 |
|
| 2655 | if (!vma_desc_test(desc, VMA_MIXEDMAP_BIT)) {
|
| 2656 | VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm));
|
| 2657 | VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT));
|
| 2658 | vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT);
|
| 2659 | }
|
| 2660 |
|
| 2661 | nr_pages = action->map_kernel.nr_pages;
|
| 2662 | end = addr + PAGE_SIZE * nr_pages;
|
| 2663 | if (!range_in_vma_desc(desc, addr, end))
|
| 2664 | return -EFAULT;
|
| 2665 |
|
| 2666 | return 0;
|
| 2667 | }
|
| 2668 | EXPORT_SYMBOL(map_kernel_pages_prepare);
|
| 2669 |
|
| 2670 | int map_kernel_pages_complete(struct vm_area_struct *vma,
|
| 2671 | struct mmap_action *action)
|
| 2672 | {
|
| 2673 | unsigned long nr_pages;
|
| 2674 |
|
| 2675 | nr_pages = action->map_kernel.nr_pages;
|
| 2676 | return insert_pages(vma, action->map_kernel.start,
|
| 2677 | action->map_kernel.pages,
|
| 2678 | &nr_pages, vma->vm_page_prot);
|
| 2679 | }
|
| 2680 | EXPORT_SYMBOL(map_kernel_pages_complete);
|
| 2681 |
|
| 2682 | |
| 2683 | * vm_insert_page - insert single page into user vma
|
| 2684 | * @vma: user vma to map to
|
| 2685 | * @addr: target user address of this page
|
| 2686 | * @page: source kernel page
|
| 2687 | *
|
| 2688 | * This allows drivers to insert individual pages they've allocated
|
| 2689 | * into a user vma. The zeropage is supported in some VMAs,
|
| 2690 | * see vm_mixed_zeropage_allowed().
|
| 2691 | *
|
| 2692 | * The page has to be a nice clean _individual_ kernel allocation.
|
| 2693 | * If you allocate a compound page, you need to have marked it as
|
| 2694 | * such (__GFP_COMP), or manually just split the page up yourself
|
| 2695 | * (see split_page()).
|
| 2696 | *
|
| 2697 | * NOTE! Traditionally this was done with "remap_pfn_range()" which
|
| 2698 | * took an arbitrary page protection parameter. This doesn't allow
|
| 2699 | * that. Your vma protection will have to be set up correctly, which
|
| 2700 | * means that if you want a shared writable mapping, you'd better
|
| 2701 | * ask for a shared writable mapping!
|
| 2702 | *
|
| 2703 | * The page does not need to be reserved.
|
| 2704 | *
|
| 2705 | * Usually this function is called from f_op->mmap() handler
|
| 2706 | * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
|
| 2707 | * Caller must set VM_MIXEDMAP on vma if it wants to call this
|
| 2708 | * function from other places, for example from page-fault handler.
|
| 2709 | *
|
| 2710 | * Return: %0 on success, negative error code otherwise.
|
| 2711 |
|
| 2712 | int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
|
| 2713 | struct page *page)
|
| 2714 | {
|
| 2715 | if (addr < vma->vm_start || addr >= vma->vm_end)
|
| 2716 | return -EFAULT;
|
| 2717 | if (!(vma->vm_flags & VM_MIXEDMAP)) {
|
| 2718 | BUG_ON(mmap_read_trylock(vma->vm_mm));
|
| 2719 | BUG_ON(vma->vm_flags & VM_PFNMAP);
|
| 2720 | vm_flags_set(vma, VM_MIXEDMAP);
|
| 2721 | }
|
| 2722 | return insert_page(vma, addr, page, vma->vm_page_prot, false);
|
| 2723 | }
|
| 2724 | EXPORT_SYMBOL(vm_insert_page);
|
| 2725 |
|
| 2726 | |
| 2727 | * __vm_map_pages - maps range of kernel pages into user vma
|
| 2728 | * @vma: user vma to map to
|
| 2729 | * @pages: pointer to array of source kernel pages
|
| 2730 | * @num: number of pages in page array
|
| 2731 | * @offset: user's requested vm_pgoff
|
| 2732 | *
|
| 2733 | * This allows drivers to map range of kernel pages into a user vma.
|
| 2734 | * The zeropage is supported in some VMAs, see
|
| 2735 | * vm_mixed_zeropage_allowed().
|
| 2736 | *
|
| 2737 | * Return: 0 on success and error code otherwise.
|
| 2738 |
|
| 2739 | static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
|
| 2740 | unsigned long num, unsigned long offset)
|
| 2741 | {
|
| 2742 | unsigned long count = vma_pages(vma);
|
| 2743 | unsigned long uaddr = vma->vm_start;
|
| 2744 |
|
| 2745 |
|
| 2746 | if (offset >= num)
|
| 2747 | return -ENXIO;
|
| 2748 |
|
| 2749 |
|
| 2750 | if (count > num - offset)
|
| 2751 | return -ENXIO;
|
| 2752 |
|
| 2753 | return vm_insert_pages(vma, uaddr, pages + offset, &count);
|
| 2754 | }
|
| 2755 |
|
| 2756 | |
| 2757 | * vm_map_pages - maps range of kernel pages starts with non zero offset
|
| 2758 | * @vma: user vma to map to
|
| 2759 | * @pages: pointer to array of source kernel pages
|
| 2760 | * @num: number of pages in page array
|
| 2761 | *
|
| 2762 | * Maps an object consisting of @num pages, catering for the user's
|
| 2763 | * requested vm_pgoff
|
| 2764 | *
|
| 2765 | * If we fail to insert any page into the vma, the function will return
|
| 2766 | * immediately leaving any previously inserted pages present. Callers
|
| 2767 | * from the mmap handler may immediately return the error as their caller
|
| 2768 | * will destroy the vma, removing any successfully inserted pages. Other
|
| 2769 | * callers should make their own arrangements for calling unmap_region().
|
| 2770 | *
|
| 2771 | * Context: Process context. Called by mmap handlers.
|
| 2772 | * Return: 0 on success and error code otherwise.
|
| 2773 |
|
| 2774 | int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
|
| 2775 | unsigned long num)
|
| 2776 | {
|
| 2777 | return __vm_map_pages(vma, pages, num, vma_start_pgoff(vma));
|
| 2778 | }
|
| 2779 | EXPORT_SYMBOL(vm_map_pages);
|
| 2780 |
|
| 2781 | |
| 2782 | * vm_map_pages_zero - map range of kernel pages starts with zero offset
|
| 2783 | * @vma: user vma to map to
|
| 2784 | * @pages: pointer to array of source kernel pages
|
| 2785 | * @num: number of pages in page array
|
| 2786 | *
|
| 2787 | * Similar to vm_map_pages(), except that it explicitly sets the offset
|
| 2788 | * to 0. This function is intended for the drivers that did not consider
|
| 2789 | * vm_pgoff.
|
| 2790 | *
|
| 2791 | * Context: Process context. Called by mmap handlers.
|
| 2792 | * Return: 0 on success and error code otherwise.
|
| 2793 |
|
| 2794 | int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
|
| 2795 | unsigned long num)
|
| 2796 | {
|
| 2797 | return __vm_map_pages(vma, pages, num, 0);
|
| 2798 | }
|
| 2799 | EXPORT_SYMBOL(vm_map_pages_zero);
|
| 2800 |
|
| 2801 | static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
|
| 2802 | unsigned long pfn, pgprot_t prot, bool mkwrite)
|
| 2803 | {
|
| 2804 | struct mm_struct *mm = vma->vm_mm;
|
| 2805 | pte_t *pte, entry;
|
| 2806 | spinlock_t *ptl;
|
| 2807 |
|
| 2808 | pte = get_locked_pte(mm, addr, &ptl);
|
| 2809 | if (!pte)
|
| 2810 | return VM_FAULT_OOM;
|
| 2811 | entry = ptep_get(pte);
|
| 2812 | if (!pte_none(entry)) {
|
| 2813 | if (mkwrite) {
|
| 2814 | |
| 2815 | * For read faults on private mappings the PFN passed
|
| 2816 | * in may not match the PFN we have mapped if the
|
| 2817 | * mapped PFN is a writeable COW page. In the mkwrite
|
| 2818 | * case we are creating a writable PTE for a shared
|
| 2819 | * mapping and we expect the PFNs to match. If they
|
| 2820 | * don't match, we are likely racing with block
|
| 2821 | * allocation and mapping invalidation so just skip the
|
| 2822 | * update.
|
| 2823 |
|
| 2824 | if (pte_pfn(entry) != pfn) {
|
| 2825 | WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
|
| 2826 | goto out_unlock;
|
| 2827 | }
|
| 2828 | entry = pte_mkyoung(entry);
|
| 2829 | entry = maybe_mkwrite(pte_mkdirty(entry), vma);
|
| 2830 | if (ptep_set_access_flags(vma, addr, pte, entry, 1))
|
| 2831 | update_mmu_cache(vma, addr, pte);
|
| 2832 | }
|
| 2833 | goto out_unlock;
|
| 2834 | }
|
| 2835 |
|
| 2836 |
|
| 2837 | entry = pte_mkspecial(pfn_pte(pfn, prot));
|
| 2838 |
|
| 2839 | if (mkwrite) {
|
| 2840 | entry = pte_mkyoung(entry);
|
| 2841 | entry = maybe_mkwrite(pte_mkdirty(entry), vma);
|
| 2842 | }
|
| 2843 |
|
| 2844 | set_pte_at(mm, addr, pte, entry);
|
| 2845 | update_mmu_cache(vma, addr, pte);
|
| 2846 |
|
| 2847 | out_unlock:
|
| 2848 | pte_unmap_unlock(pte, ptl);
|
| 2849 | return VM_FAULT_NOPAGE;
|
| 2850 | }
|
| 2851 |
|
| 2852 | |
| 2853 | * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
|
| 2854 | * @vma: user vma to map to
|
| 2855 | * @addr: target user address of this page
|
| 2856 | * @pfn: source kernel pfn
|
| 2857 | * @pgprot: pgprot flags for the inserted page
|
| 2858 | *
|
| 2859 | * This is exactly like vmf_insert_pfn(), except that it allows drivers
|
| 2860 | * to override pgprot on a per-page basis.
|
| 2861 | *
|
| 2862 | * This only makes sense for IO mappings, and it makes no sense for
|
| 2863 | * COW mappings. In general, using multiple vmas is preferable;
|
| 2864 | * vmf_insert_pfn_prot should only be used if using multiple VMAs is
|
| 2865 | * impractical.
|
| 2866 | *
|
| 2867 | * pgprot typically only differs from @vma->vm_page_prot when drivers set
|
| 2868 | * caching- and encryption bits different than those of @vma->vm_page_prot,
|
| 2869 | * because the caching- or encryption mode may not be known at mmap() time.
|
| 2870 | *
|
| 2871 | * This is ok as long as @vma->vm_page_prot is not used by the core vm
|
| 2872 | * to set caching and encryption bits for those vmas (except for COW pages).
|
| 2873 | * This is ensured by core vm only modifying these page table entries using
|
| 2874 | * functions that don't touch caching- or encryption bits, using pte_modify()
|
| 2875 | * if needed. (See for example mprotect()).
|
| 2876 | *
|
| 2877 | * Also when new page-table entries are created, this is only done using the
|
| 2878 | * fault() callback, and never using the value of vma->vm_page_prot,
|
| 2879 | * except for page-table entries that point to anonymous pages as the result
|
| 2880 | * of COW.
|
| 2881 | *
|
| 2882 | * Context: Process context. May allocate using %GFP_KERNEL.
|
| 2883 | * Return: vm_fault_t value.
|
| 2884 |
|
| 2885 | vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
|
| 2886 | unsigned long pfn, pgprot_t pgprot)
|
| 2887 | {
|
| 2888 | |
| 2889 | * Technically, architectures with pte_special can avoid all these
|
| 2890 | * restrictions (same for remap_pfn_range). However we would like
|
| 2891 | * consistency in testing and feature parity among all, so we should
|
| 2892 | * try to keep these invariants in place for everybody.
|
| 2893 |
|
| 2894 | BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
|
| 2895 | BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
|
| 2896 | (VM_PFNMAP|VM_MIXEDMAP));
|
| 2897 | BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
|
| 2898 | BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
|
| 2899 |
|
| 2900 | if (addr < vma->vm_start || addr >= vma->vm_end)
|
| 2901 | return VM_FAULT_SIGBUS;
|
| 2902 |
|
| 2903 | if (!pfn_modify_allowed(pfn, pgprot))
|
| 2904 | return VM_FAULT_SIGBUS;
|
| 2905 |
|
| 2906 | pfnmap_setup_cachemode_pfn(pfn, &pgprot);
|
| 2907 |
|
| 2908 | return insert_pfn(vma, addr, pfn, pgprot, false);
|
| 2909 | }
|
| 2910 | EXPORT_SYMBOL(vmf_insert_pfn_prot);
|
| 2911 |
|
| 2912 | |
| 2913 | * vmf_insert_pfn - insert single pfn into user vma
|
| 2914 | * @vma: user vma to map to
|
| 2915 | * @addr: target user address of this page
|
| 2916 | * @pfn: source kernel pfn
|
| 2917 | *
|
| 2918 | * Similar to vm_insert_page, this allows drivers to insert individual pages
|
| 2919 | * they've allocated into a user vma. Same comments apply.
|
| 2920 | *
|
| 2921 | * This function should only be called from a vm_ops->fault handler, and
|
| 2922 | * in that case the handler should return the result of this function.
|
| 2923 | *
|
| 2924 | * vma cannot be a COW mapping.
|
| 2925 | *
|
| 2926 | * As this is called only for pages that do not currently exist, we
|
| 2927 | * do not need to flush old virtual caches or the TLB.
|
| 2928 | *
|
| 2929 | * Context: Process context. May allocate using %GFP_KERNEL.
|
| 2930 | * Return: vm_fault_t value.
|
| 2931 |
|
| 2932 | vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
|
| 2933 | unsigned long pfn)
|
| 2934 | {
|
| 2935 | return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
|
| 2936 | }
|
| 2937 | EXPORT_SYMBOL(vmf_insert_pfn);
|
| 2938 |
|
| 2939 | static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn,
|
| 2940 | bool mkwrite)
|
| 2941 | {
|
| 2942 | if (unlikely(is_zero_pfn(pfn)) &&
|
| 2943 | (mkwrite || !vm_mixed_zeropage_allowed(vma)))
|
| 2944 | return false;
|
| 2945 |
|
| 2946 | if (vma->vm_flags & VM_MIXEDMAP)
|
| 2947 | return true;
|
| 2948 | if (is_zero_pfn(pfn))
|
| 2949 | return true;
|
| 2950 | return false;
|
| 2951 | }
|
| 2952 |
|
| 2953 | static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
|
| 2954 | unsigned long addr, unsigned long pfn, bool mkwrite)
|
| 2955 | {
|
| 2956 | pgprot_t pgprot = vma->vm_page_prot;
|
| 2957 | int err;
|
| 2958 |
|
| 2959 | if (!vm_mixed_ok(vma, pfn, mkwrite))
|
| 2960 | return VM_FAULT_SIGBUS;
|
| 2961 |
|
| 2962 | if (addr < vma->vm_start || addr >= vma->vm_end)
|
| 2963 | return VM_FAULT_SIGBUS;
|
| 2964 |
|
| 2965 | pfnmap_setup_cachemode_pfn(pfn, &pgprot);
|
| 2966 |
|
| 2967 | if (!pfn_modify_allowed(pfn, pgprot))
|
| 2968 | return VM_FAULT_SIGBUS;
|
| 2969 |
|
| 2970 | |
| 2971 | * If we don't have pte special, then we have to use the pfn_valid()
|
| 2972 | * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
|
| 2973 | * refcount the page if pfn_valid is true (hence insert_page rather
|
| 2974 | * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
|
| 2975 | * without pte special, it would there be refcounted as a normal page.
|
| 2976 |
|
| 2977 | if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pfn_valid(pfn)) {
|
| 2978 | struct page *page;
|
| 2979 |
|
| 2980 | |
| 2981 | * At this point we are committed to insert_page()
|
| 2982 | * regardless of whether the caller specified flags that
|
| 2983 | * result in pfn_t_has_page() == false.
|
| 2984 |
|
| 2985 | page = pfn_to_page(pfn);
|
| 2986 | err = insert_page(vma, addr, page, pgprot, mkwrite);
|
| 2987 | } else {
|
| 2988 | return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
|
| 2989 | }
|
| 2990 |
|
| 2991 | if (err == -ENOMEM)
|
| 2992 | return VM_FAULT_OOM;
|
| 2993 | if (err < 0 && err != -EBUSY)
|
| 2994 | return VM_FAULT_SIGBUS;
|
| 2995 |
|
| 2996 | return VM_FAULT_NOPAGE;
|
| 2997 | }
|
| 2998 |
|
| 2999 | vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page,
|
| 3000 | bool write)
|
| 3001 | {
|
| 3002 | pgprot_t pgprot = vmf->vma->vm_page_prot;
|
| 3003 | unsigned long addr = vmf->address;
|
| 3004 | int err;
|
| 3005 |
|
| 3006 | if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end)
|
| 3007 | return VM_FAULT_SIGBUS;
|
| 3008 |
|
| 3009 | err = insert_page(vmf->vma, addr, page, pgprot, write);
|
| 3010 | if (err == -ENOMEM)
|
| 3011 | return VM_FAULT_OOM;
|
| 3012 | if (err < 0 && err != -EBUSY)
|
| 3013 | return VM_FAULT_SIGBUS;
|
| 3014 |
|
| 3015 | return VM_FAULT_NOPAGE;
|
| 3016 | }
|
| 3017 | EXPORT_SYMBOL_GPL(vmf_insert_page_mkwrite);
|
| 3018 |
|
| 3019 | vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
|
| 3020 | unsigned long pfn)
|
| 3021 | {
|
| 3022 | return __vm_insert_mixed(vma, addr, pfn, false);
|
| 3023 | }
|
| 3024 | EXPORT_SYMBOL(vmf_insert_mixed);
|
| 3025 |
|
| 3026 | |
| 3027 | * If the insertion of PTE failed because someone else already added a
|
| 3028 | * different entry in the mean time, we treat that as success as we assume
|
| 3029 | * the same entry was actually inserted.
|
| 3030 |
|
| 3031 | vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
|
| 3032 | unsigned long addr, unsigned long pfn)
|
| 3033 | {
|
| 3034 | return __vm_insert_mixed(vma, addr, pfn, true);
|
| 3035 | }
|
| 3036 |
|
| 3037 | |
| 3038 | * maps a range of physical memory into the requested pages. the old
|
| 3039 | * mappings are removed. any references to nonexistent pages results
|
| 3040 | * in null mappings (currently treated as "copy-on-access")
|
| 3041 |
|
| 3042 | static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
|
| 3043 | unsigned long addr, unsigned long end,
|
| 3044 | unsigned long pfn, pgprot_t prot)
|
| 3045 | {
|
| 3046 | pte_t *pte, *mapped_pte;
|
| 3047 | spinlock_t *ptl;
|
| 3048 | int err = 0;
|
| 3049 |
|
| 3050 | mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
|
| 3051 | if (!pte)
|
| 3052 | return -ENOMEM;
|
| 3053 | lazy_mmu_mode_enable();
|
| 3054 | do {
|
| 3055 | BUG_ON(!pte_none(ptep_get(pte)));
|
| 3056 | if (!pfn_modify_allowed(pfn, prot)) {
|
| 3057 | err = -EACCES;
|
| 3058 | break;
|
| 3059 | }
|
| 3060 | set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
|
| 3061 | pfn++;
|
| 3062 | } while (pte++, addr += PAGE_SIZE, addr != end);
|
| 3063 | lazy_mmu_mode_disable();
|
| 3064 | pte_unmap_unlock(mapped_pte, ptl);
|
| 3065 | return err;
|
| 3066 | }
|
| 3067 |
|
| 3068 | static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
|
| 3069 | unsigned long addr, unsigned long end,
|
| 3070 | unsigned long pfn, pgprot_t prot)
|
| 3071 | {
|
| 3072 | pmd_t *pmd;
|
| 3073 | unsigned long next;
|
| 3074 | int err;
|
| 3075 |
|
| 3076 | pfn -= addr >> PAGE_SHIFT;
|
| 3077 | pmd = pmd_alloc(mm, pud, addr);
|
| 3078 | if (!pmd)
|
| 3079 | return -ENOMEM;
|
| 3080 | VM_BUG_ON(pmd_trans_huge(*pmd));
|
| 3081 | do {
|
| 3082 | next = pmd_addr_end(addr, end);
|
| 3083 | err = remap_pte_range(mm, pmd, addr, next,
|
| 3084 | pfn + (addr >> PAGE_SHIFT), prot);
|
| 3085 | if (err)
|
| 3086 | return err;
|
| 3087 | } while (pmd++, addr = next, addr != end);
|
| 3088 | return 0;
|
| 3089 | }
|
| 3090 |
|
| 3091 | static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
|
| 3092 | unsigned long addr, unsigned long end,
|
| 3093 | unsigned long pfn, pgprot_t prot)
|
| 3094 | {
|
| 3095 | pud_t *pud;
|
| 3096 | unsigned long next;
|
| 3097 | int err;
|
| 3098 |
|
| 3099 | pfn -= addr >> PAGE_SHIFT;
|
| 3100 | pud = pud_alloc(mm, p4d, addr);
|
| 3101 | if (!pud)
|
| 3102 | return -ENOMEM;
|
| 3103 | do {
|
| 3104 | next = pud_addr_end(addr, end);
|
| 3105 | err = remap_pmd_range(mm, pud, addr, next,
|
| 3106 | pfn + (addr >> PAGE_SHIFT), prot);
|
| 3107 | if (err)
|
| 3108 | return err;
|
| 3109 | } while (pud++, addr = next, addr != end);
|
| 3110 | return 0;
|
| 3111 | }
|
| 3112 |
|
| 3113 | static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
|
| 3114 | unsigned long addr, unsigned long end,
|
| 3115 | unsigned long pfn, pgprot_t prot)
|
| 3116 | {
|
| 3117 | p4d_t *p4d;
|
| 3118 | unsigned long next;
|
| 3119 | int err;
|
| 3120 |
|
| 3121 | pfn -= addr >> PAGE_SHIFT;
|
| 3122 | p4d = p4d_alloc(mm, pgd, addr);
|
| 3123 | if (!p4d)
|
| 3124 | return -ENOMEM;
|
| 3125 | do {
|
| 3126 | next = p4d_addr_end(addr, end);
|
| 3127 | err = remap_pud_range(mm, p4d, addr, next,
|
| 3128 | pfn + (addr >> PAGE_SHIFT), prot);
|
| 3129 | if (err)
|
| 3130 | return err;
|
| 3131 | } while (p4d++, addr = next, addr != end);
|
| 3132 | return 0;
|
| 3133 | }
|
| 3134 |
|
| 3135 | static int get_remap_pgoff(bool is_cow, unsigned long addr,
|
| 3136 | unsigned long end, unsigned long vm_start, unsigned long vm_end,
|
| 3137 | unsigned long pfn, pgoff_t *vm_pgoff_p)
|
| 3138 | {
|
| 3139 | |
| 3140 | * There's a horrible special case to handle copy-on-write
|
| 3141 | * behaviour that some programs depend on. We mark the "original"
|
| 3142 | * un-COW'ed pages by matching them up with "vma->vm_pgoff".
|
| 3143 | * See vm_normal_page() for details.
|
| 3144 |
|
| 3145 | if (is_cow) {
|
| 3146 | if (addr != vm_start || end != vm_end)
|
| 3147 | return -EINVAL;
|
| 3148 | *vm_pgoff_p = pfn;
|
| 3149 | }
|
| 3150 |
|
| 3151 | return 0;
|
| 3152 | }
|
| 3153 |
|
| 3154 | static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr,
|
| 3155 | unsigned long pfn, unsigned long size, pgprot_t prot)
|
| 3156 | {
|
| 3157 | pgd_t *pgd;
|
| 3158 | unsigned long next;
|
| 3159 | unsigned long end = addr + PAGE_ALIGN(size);
|
| 3160 | struct mm_struct *mm = vma->vm_mm;
|
| 3161 | int err;
|
| 3162 |
|
| 3163 | if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
|
| 3164 | return -EINVAL;
|
| 3165 |
|
| 3166 | VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS));
|
| 3167 |
|
| 3168 | BUG_ON(addr >= end);
|
| 3169 | pfn -= addr >> PAGE_SHIFT;
|
| 3170 | pgd = pgd_offset(mm, addr);
|
| 3171 | flush_cache_range(vma, addr, end);
|
| 3172 | do {
|
| 3173 | next = pgd_addr_end(addr, end);
|
| 3174 | err = remap_p4d_range(mm, pgd, addr, next,
|
| 3175 | pfn + (addr >> PAGE_SHIFT), prot);
|
| 3176 | if (err)
|
| 3177 | return err;
|
| 3178 | } while (pgd++, addr = next, addr != end);
|
| 3179 |
|
| 3180 | return 0;
|
| 3181 | }
|
| 3182 |
|
| 3183 | |
| 3184 | * Variant of remap_pfn_range that does not call track_pfn_remap. The caller
|
| 3185 | * must have pre-validated the caching bits of the pgprot_t.
|
| 3186 |
|
| 3187 | static int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
|
| 3188 | unsigned long pfn, unsigned long size, pgprot_t prot)
|
| 3189 | {
|
| 3190 | int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);
|
| 3191 |
|
| 3192 | if (!error)
|
| 3193 | return 0;
|
| 3194 |
|
| 3195 | |
| 3196 | * A partial pfn range mapping is dangerous: it does not
|
| 3197 | * maintain page reference counts, and callers may free
|
| 3198 | * pages due to the error. So zap it early.
|
| 3199 |
|
| 3200 | zap_vma_range(vma, addr, size);
|
| 3201 | return error;
|
| 3202 | }
|
| 3203 |
|
| 3204 | #ifdef __HAVE_PFNMAP_TRACKING
|
| 3205 | static inline struct pfnmap_track_ctx *pfnmap_track_ctx_alloc(unsigned long pfn,
|
| 3206 | unsigned long size, pgprot_t *prot)
|
| 3207 | {
|
| 3208 | struct pfnmap_track_ctx *ctx;
|
| 3209 |
|
| 3210 | if (pfnmap_track(pfn, size, prot))
|
| 3211 | return ERR_PTR(-EINVAL);
|
| 3212 |
|
| 3213 | ctx = kmalloc_obj(*ctx);
|
| 3214 | if (unlikely(!ctx)) {
|
| 3215 | pfnmap_untrack(pfn, size);
|
| 3216 | return ERR_PTR(-ENOMEM);
|
| 3217 | }
|
| 3218 |
|
| 3219 | ctx->pfn = pfn;
|
| 3220 | ctx->size = size;
|
| 3221 | kref_init(&ctx->kref);
|
| 3222 | return ctx;
|
| 3223 | }
|
| 3224 |
|
| 3225 | void pfnmap_track_ctx_release(struct kref *ref)
|
| 3226 | {
|
| 3227 | struct pfnmap_track_ctx *ctx = container_of(ref, struct pfnmap_track_ctx, kref);
|
| 3228 |
|
| 3229 | pfnmap_untrack(ctx->pfn, ctx->size);
|
| 3230 | kfree(ctx);
|
| 3231 | }
|
| 3232 |
|
| 3233 | static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr,
|
| 3234 | unsigned long pfn, unsigned long size, pgprot_t prot)
|
| 3235 | {
|
| 3236 | struct pfnmap_track_ctx *ctx = NULL;
|
| 3237 | int err;
|
| 3238 |
|
| 3239 | size = PAGE_ALIGN(size);
|
| 3240 |
|
| 3241 | |
| 3242 | * If we cover the full VMA, we'll perform actual tracking, and
|
| 3243 | * remember to untrack when the last reference to our tracking
|
| 3244 | * context from a VMA goes away. We'll keep tracking the whole pfn
|
| 3245 | * range even during VMA splits and partial unmapping.
|
| 3246 | *
|
| 3247 | * If we only cover parts of the VMA, we'll only setup the cachemode
|
| 3248 | * in the pgprot for the pfn range.
|
| 3249 |
|
| 3250 | if (addr == vma->vm_start && addr + size == vma->vm_end) {
|
| 3251 | if (vma->pfnmap_track_ctx)
|
| 3252 | return -EINVAL;
|
| 3253 | ctx = pfnmap_track_ctx_alloc(pfn, size, &prot);
|
| 3254 | if (IS_ERR(ctx))
|
| 3255 | return PTR_ERR(ctx);
|
| 3256 | } else if (pfnmap_setup_cachemode(pfn, size, &prot)) {
|
| 3257 | return -EINVAL;
|
| 3258 | }
|
| 3259 |
|
| 3260 | err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
|
| 3261 | if (ctx) {
|
| 3262 | if (err)
|
| 3263 | kref_put(&ctx->kref, pfnmap_track_ctx_release);
|
| 3264 | else
|
| 3265 | vma->pfnmap_track_ctx = ctx;
|
| 3266 | }
|
| 3267 | return err;
|
| 3268 | }
|
| 3269 |
|
| 3270 | static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
|
| 3271 | unsigned long pfn, unsigned long size, pgprot_t prot)
|
| 3272 | {
|
| 3273 | return remap_pfn_range_track(vma, addr, pfn, size, prot);
|
| 3274 | }
|
| 3275 | #else
|
| 3276 | static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
|
| 3277 | unsigned long pfn, unsigned long size, pgprot_t prot)
|
| 3278 | {
|
| 3279 | return remap_pfn_range_notrack(vma, addr, pfn, size, prot);
|
| 3280 | }
|
| 3281 | #endif
|
| 3282 |
|
| 3283 | int remap_pfn_range_prepare(struct vm_area_desc *desc)
|
| 3284 | {
|
| 3285 | const struct mmap_action *action = &desc->action;
|
| 3286 | const unsigned long start = action->remap.start;
|
| 3287 | const unsigned long end = start + action->remap.size;
|
| 3288 | const unsigned long pfn = action->remap.start_pfn;
|
| 3289 | const bool is_cow = vma_desc_is_cow_mapping(desc);
|
| 3290 | int err;
|
| 3291 |
|
| 3292 | if (!range_in_vma_desc(desc, start, end))
|
| 3293 | return -EFAULT;
|
| 3294 |
|
| 3295 | err = get_remap_pgoff(is_cow, start, end, desc->start, desc->end, pfn,
|
| 3296 | &desc->pgoff);
|
| 3297 | if (err)
|
| 3298 | return err;
|
| 3299 |
|
| 3300 | vma_desc_set_flags_mask(desc, VMA_REMAP_FLAGS);
|
| 3301 | return 0;
|
| 3302 | }
|
| 3303 |
|
| 3304 | static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma,
|
| 3305 | unsigned long addr, unsigned long pfn,
|
| 3306 | unsigned long size)
|
| 3307 | {
|
| 3308 | const unsigned long end = addr + PAGE_ALIGN(size);
|
| 3309 | const bool is_cow = vma_is_cow_mapping(vma);
|
| 3310 | int err;
|
| 3311 |
|
| 3312 | err = get_remap_pgoff(is_cow, addr, end, vma->vm_start, vma->vm_end,
|
| 3313 | pfn, &vma->vm_pgoff);
|
| 3314 | if (err)
|
| 3315 | return err;
|
| 3316 |
|
| 3317 | vma_set_flags_mask(vma, VMA_REMAP_FLAGS);
|
| 3318 | return 0;
|
| 3319 | }
|
| 3320 |
|
| 3321 | |
| 3322 | * remap_pfn_range - remap kernel memory to userspace
|
| 3323 | * @vma: user vma to map to
|
| 3324 | * @addr: target page aligned user address to start at
|
| 3325 | * @pfn: page frame number of kernel physical memory address
|
| 3326 | * @size: size of mapping area
|
| 3327 | * @prot: page protection flags for this mapping
|
| 3328 | *
|
| 3329 | * Note: this is only safe if the mm semaphore is held when called.
|
| 3330 | *
|
| 3331 | * Return: %0 on success, negative error code otherwise.
|
| 3332 |
|
| 3333 | int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
|
| 3334 | unsigned long pfn, unsigned long size, pgprot_t prot)
|
| 3335 | {
|
| 3336 | int err;
|
| 3337 |
|
| 3338 | err = remap_pfn_range_prepare_vma(vma, addr, pfn, size);
|
| 3339 | if (err)
|
| 3340 | return err;
|
| 3341 |
|
| 3342 | return do_remap_pfn_range(vma, addr, pfn, size, prot);
|
| 3343 | }
|
| 3344 | EXPORT_SYMBOL(remap_pfn_range);
|
| 3345 |
|
| 3346 | int remap_pfn_range_complete(struct vm_area_struct *vma,
|
| 3347 | struct mmap_action *action)
|
| 3348 | {
|
| 3349 | const unsigned long start = action->remap.start;
|
| 3350 | const unsigned long pfn = action->remap.start_pfn;
|
| 3351 | const unsigned long size = action->remap.size;
|
| 3352 | const pgprot_t prot = action->remap.pgprot;
|
| 3353 |
|
| 3354 | return do_remap_pfn_range(vma, start, pfn, size, prot);
|
| 3355 | }
|
| 3356 |
|
| 3357 | static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff,
|
| 3358 | phys_addr_t start_phys, unsigned long size,
|
| 3359 | unsigned long *pfnp)
|
| 3360 | {
|
| 3361 | unsigned long pfn, pages;
|
| 3362 |
|
| 3363 |
|
| 3364 | if (start_phys + size < start_phys)
|
| 3365 | return -EINVAL;
|
| 3366 | |
| 3367 | * You *really* shouldn't map things that aren't page-aligned,
|
| 3368 | * but we've historically allowed it because IO memory might
|
| 3369 | * just have smaller alignment.
|
| 3370 |
|
| 3371 | size += start_phys & ~PAGE_MASK;
|
| 3372 | pfn = start_phys >> PAGE_SHIFT;
|
| 3373 | pages = (size + ~PAGE_MASK) >> PAGE_SHIFT;
|
| 3374 | if (pfn + pages < pfn)
|
| 3375 | return -EINVAL;
|
| 3376 |
|
| 3377 |
|
| 3378 | if (vm_pgoff > pages)
|
| 3379 | return -EINVAL;
|
| 3380 | pfn += vm_pgoff;
|
| 3381 | pages -= vm_pgoff;
|
| 3382 |
|
| 3383 |
|
| 3384 | if ((vm_len >> PAGE_SHIFT) > pages)
|
| 3385 | return -EINVAL;
|
| 3386 |
|
| 3387 | *pfnp = pfn;
|
| 3388 | return 0;
|
| 3389 | }
|
| 3390 |
|
| 3391 | int simple_ioremap_prepare(struct vm_area_desc *desc)
|
| 3392 | {
|
| 3393 | struct mmap_action *action = &desc->action;
|
| 3394 | const phys_addr_t start = action->simple_ioremap.start_phys_addr;
|
| 3395 | const unsigned long size = action->simple_ioremap.size;
|
| 3396 | unsigned long pfn;
|
| 3397 | int err;
|
| 3398 |
|
| 3399 | err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff,
|
| 3400 | start, size, &pfn);
|
| 3401 | if (err)
|
| 3402 | return err;
|
| 3403 |
|
| 3404 |
|
| 3405 | mmap_action_ioremap_full(desc, pfn);
|
| 3406 | return io_remap_pfn_range_prepare(desc);
|
| 3407 | }
|
| 3408 |
|
| 3409 | |
| 3410 | * vm_iomap_memory - remap memory to userspace
|
| 3411 | * @vma: user vma to map to
|
| 3412 | * @start: start of the physical memory to be mapped
|
| 3413 | * @len: size of area
|
| 3414 | *
|
| 3415 | * This is a simplified io_remap_pfn_range() for common driver use. The
|
| 3416 | * driver just needs to give us the physical memory range to be mapped,
|
| 3417 | * we'll figure out the rest from the vma information.
|
| 3418 | *
|
| 3419 | * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
|
| 3420 | * whatever write-combining details or similar.
|
| 3421 | *
|
| 3422 | * Return: %0 on success, negative error code otherwise.
|
| 3423 |
|
| 3424 | int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
|
| 3425 | {
|
| 3426 | const unsigned long vm_start = vma->vm_start;
|
| 3427 | const unsigned long vm_end = vma->vm_end;
|
| 3428 | const unsigned long vm_len = vm_end - vm_start;
|
| 3429 | unsigned long pfn;
|
| 3430 | int err;
|
| 3431 |
|
| 3432 | err = __simple_ioremap_prep(vm_len, vma_start_pgoff(vma), start, len,
|
| 3433 | &pfn);
|
| 3434 | if (err)
|
| 3435 | return err;
|
| 3436 |
|
| 3437 |
|
| 3438 | return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
|
| 3439 | }
|
| 3440 | EXPORT_SYMBOL(vm_iomap_memory);
|
| 3441 |
|
| 3442 | static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
|
| 3443 | unsigned long addr, unsigned long end,
|
| 3444 | pte_fn_t fn, void *data, bool create,
|
| 3445 | pgtbl_mod_mask *mask)
|
| 3446 | {
|
| 3447 | pte_t *pte, *mapped_pte;
|
| 3448 | int err = 0;
|
| 3449 | spinlock_t *ptl;
|
| 3450 |
|
| 3451 | if (create) {
|
| 3452 | mapped_pte = pte = (mm == &init_mm) ?
|
| 3453 | pte_alloc_kernel_track(pmd, addr, mask) :
|
| 3454 | pte_alloc_map_lock(mm, pmd, addr, &ptl);
|
| 3455 | if (!pte)
|
| 3456 | return -ENOMEM;
|
| 3457 | } else {
|
| 3458 | mapped_pte = pte = (mm == &init_mm) ?
|
| 3459 | pte_offset_kernel(pmd, addr) :
|
| 3460 | pte_offset_map_lock(mm, pmd, addr, &ptl);
|
| 3461 | if (!pte)
|
| 3462 | return -EINVAL;
|
| 3463 | }
|
| 3464 |
|
| 3465 | lazy_mmu_mode_enable();
|
| 3466 |
|
| 3467 | if (fn) {
|
| 3468 | do {
|
| 3469 | if (create || !pte_none(ptep_get(pte))) {
|
| 3470 | err = fn(pte, addr, data);
|
| 3471 | if (err)
|
| 3472 | break;
|
| 3473 | }
|
| 3474 | } while (pte++, addr += PAGE_SIZE, addr != end);
|
| 3475 | }
|
| 3476 | *mask |= PGTBL_PTE_MODIFIED;
|
| 3477 |
|
| 3478 | lazy_mmu_mode_disable();
|
| 3479 |
|
| 3480 | if (mm != &init_mm)
|
| 3481 | pte_unmap_unlock(mapped_pte, ptl);
|
| 3482 | return err;
|
| 3483 | }
|
| 3484 |
|
| 3485 | static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
|
| 3486 | unsigned long addr, unsigned long end,
|
| 3487 | pte_fn_t fn, void *data, bool create,
|
| 3488 | pgtbl_mod_mask *mask)
|
| 3489 | {
|
| 3490 | pmd_t *pmd;
|
| 3491 | unsigned long next;
|
| 3492 | int err = 0;
|
| 3493 |
|
| 3494 | BUG_ON(pud_leaf(*pud));
|
| 3495 |
|
| 3496 | if (create) {
|
| 3497 | pmd = pmd_alloc_track(mm, pud, addr, mask);
|
| 3498 | if (!pmd)
|
| 3499 | return -ENOMEM;
|
| 3500 | } else {
|
| 3501 | pmd = pmd_offset(pud, addr);
|
| 3502 | }
|
| 3503 | do {
|
| 3504 | next = pmd_addr_end(addr, end);
|
| 3505 | if (pmd_none(*pmd) && !create)
|
| 3506 | continue;
|
| 3507 | if (WARN_ON_ONCE(pmd_leaf(*pmd)))
|
| 3508 | return -EINVAL;
|
| 3509 | if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
|
| 3510 | if (!create)
|
| 3511 | continue;
|
| 3512 | pmd_clear_bad(pmd);
|
| 3513 | }
|
| 3514 | err = apply_to_pte_range(mm, pmd, addr, next,
|
| 3515 | fn, data, create, mask);
|
| 3516 | if (err)
|
| 3517 | break;
|
| 3518 | } while (pmd++, addr = next, addr != end);
|
| 3519 |
|
| 3520 | return err;
|
| 3521 | }
|
| 3522 |
|
| 3523 | static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
|
| 3524 | unsigned long addr, unsigned long end,
|
| 3525 | pte_fn_t fn, void *data, bool create,
|
| 3526 | pgtbl_mod_mask *mask)
|
| 3527 | {
|
| 3528 | pud_t *pud;
|
| 3529 | unsigned long next;
|
| 3530 | int err = 0;
|
| 3531 |
|
| 3532 | if (create) {
|
| 3533 | pud = pud_alloc_track(mm, p4d, addr, mask);
|
| 3534 | if (!pud)
|
| 3535 | return -ENOMEM;
|
| 3536 | } else {
|
| 3537 | pud = pud_offset(p4d, addr);
|
| 3538 | }
|
| 3539 | do {
|
| 3540 | next = pud_addr_end(addr, end);
|
| 3541 | if (pud_none(*pud) && !create)
|
| 3542 | continue;
|
| 3543 | if (WARN_ON_ONCE(pud_leaf(*pud)))
|
| 3544 | return -EINVAL;
|
| 3545 | if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
|
| 3546 | if (!create)
|
| 3547 | continue;
|
| 3548 | pud_clear_bad(pud);
|
| 3549 | }
|
| 3550 | err = apply_to_pmd_range(mm, pud, addr, next,
|
| 3551 | fn, data, create, mask);
|
| 3552 | if (err)
|
| 3553 | break;
|
| 3554 | } while (pud++, addr = next, addr != end);
|
| 3555 |
|
| 3556 | return err;
|
| 3557 | }
|
| 3558 |
|
| 3559 | static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
|
| 3560 | unsigned long addr, unsigned long end,
|
| 3561 | pte_fn_t fn, void *data, bool create,
|
| 3562 | pgtbl_mod_mask *mask)
|
| 3563 | {
|
| 3564 | p4d_t *p4d;
|
| 3565 | unsigned long next;
|
| 3566 | int err = 0;
|
| 3567 |
|
| 3568 | if (create) {
|
| 3569 | p4d = p4d_alloc_track(mm, pgd, addr, mask);
|
| 3570 | if (!p4d)
|
| 3571 | return -ENOMEM;
|
| 3572 | } else {
|
| 3573 | p4d = p4d_offset(pgd, addr);
|
| 3574 | }
|
| 3575 | do {
|
| 3576 | next = p4d_addr_end(addr, end);
|
| 3577 | if (p4d_none(*p4d) && !create)
|
| 3578 | continue;
|
| 3579 | if (WARN_ON_ONCE(p4d_leaf(*p4d)))
|
| 3580 | return -EINVAL;
|
| 3581 | if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
|
| 3582 | if (!create)
|
| 3583 | continue;
|
| 3584 | p4d_clear_bad(p4d);
|
| 3585 | }
|
| 3586 | err = apply_to_pud_range(mm, p4d, addr, next,
|
| 3587 | fn, data, create, mask);
|
| 3588 | if (err)
|
| 3589 | break;
|
| 3590 | } while (p4d++, addr = next, addr != end);
|
| 3591 |
|
| 3592 | return err;
|
| 3593 | }
|
| 3594 |
|
| 3595 | static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
|
| 3596 | unsigned long size, pte_fn_t fn,
|
| 3597 | void *data, bool create)
|
| 3598 | {
|
| 3599 | pgd_t *pgd;
|
| 3600 | unsigned long start = addr, next;
|
| 3601 | unsigned long end = addr + size;
|
| 3602 | pgtbl_mod_mask mask = 0;
|
| 3603 | int err = 0;
|
| 3604 |
|
| 3605 | if (WARN_ON(addr >= end))
|
| 3606 | return -EINVAL;
|
| 3607 |
|
| 3608 | pgd = pgd_offset(mm, addr);
|
| 3609 | do {
|
| 3610 | next = pgd_addr_end(addr, end);
|
| 3611 | if (pgd_none(*pgd) && !create)
|
| 3612 | continue;
|
| 3613 | if (WARN_ON_ONCE(pgd_leaf(*pgd))) {
|
| 3614 | err = -EINVAL;
|
| 3615 | break;
|
| 3616 | }
|
| 3617 | if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
|
| 3618 | if (!create)
|
| 3619 | continue;
|
| 3620 | pgd_clear_bad(pgd);
|
| 3621 | }
|
| 3622 | err = apply_to_p4d_range(mm, pgd, addr, next,
|
| 3623 | fn, data, create, &mask);
|
| 3624 | if (err)
|
| 3625 | break;
|
| 3626 | } while (pgd++, addr = next, addr != end);
|
| 3627 |
|
| 3628 | if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
|
| 3629 | arch_sync_kernel_mappings(start, start + size);
|
| 3630 |
|
| 3631 | return err;
|
| 3632 | }
|
| 3633 |
|
| 3634 | |
| 3635 | * Scan a region of virtual memory, filling in page tables as necessary
|
| 3636 | * and calling a provided function on each leaf page table.
|
| 3637 |
|
| 3638 | int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
|
| 3639 | unsigned long size, pte_fn_t fn, void *data)
|
| 3640 | {
|
| 3641 | return __apply_to_page_range(mm, addr, size, fn, data, true);
|
| 3642 | }
|
| 3643 | EXPORT_SYMBOL_GPL(apply_to_page_range);
|
| 3644 |
|
| 3645 | |
| 3646 | * Scan a region of virtual memory, calling a provided function on
|
| 3647 | * each leaf page table where it exists.
|
| 3648 | *
|
| 3649 | * Unlike apply_to_page_range, this does _not_ fill in page tables
|
| 3650 | * where they are absent.
|
| 3651 |
|
| 3652 | int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
|
| 3653 | unsigned long size, pte_fn_t fn, void *data)
|
| 3654 | {
|
| 3655 | return __apply_to_page_range(mm, addr, size, fn, data, false);
|
| 3656 | }
|
| 3657 |
|
| 3658 | |
| 3659 | * handle_pte_fault chooses page fault handler according to an entry which was
|
| 3660 | * read non-atomically. Before making any commitment, on those architectures
|
| 3661 | * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
|
| 3662 | * parts, do_swap_page must check under lock before unmapping the pte and
|
| 3663 | * proceeding (but do_wp_page is only called after already making such a check;
|
| 3664 | * and do_anonymous_page can safely check later on).
|
| 3665 |
|
| 3666 | static inline int pte_unmap_same(struct vm_fault *vmf)
|
| 3667 | {
|
| 3668 | int same = 1;
|
| 3669 | #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
|
| 3670 | if (sizeof(pte_t) > sizeof(unsigned long)) {
|
| 3671 | spin_lock(vmf->ptl);
|
| 3672 | same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
|
| 3673 | spin_unlock(vmf->ptl);
|
| 3674 | }
|
| 3675 | #endif
|
| 3676 | pte_unmap(vmf->pte);
|
| 3677 | vmf->pte = NULL;
|
| 3678 | return same;
|
| 3679 | }
|
| 3680 |
|
| 3681 | |
| 3682 | * Return:
|
| 3683 | * 0: copied succeeded
|
| 3684 | * -EHWPOISON: copy failed due to hwpoison in source page
|
| 3685 | * -EAGAIN: copied failed (some other reason)
|
| 3686 |
|
| 3687 | static inline int __wp_page_copy_user(struct page *dst, struct page *src,
|
| 3688 | struct vm_fault *vmf)
|
| 3689 | {
|
| 3690 | int ret;
|
| 3691 | void *kaddr;
|
| 3692 | void __user *uaddr;
|
| 3693 | struct vm_area_struct *vma = vmf->vma;
|
| 3694 | struct mm_struct *mm = vma->vm_mm;
|
| 3695 | unsigned long addr = vmf->address;
|
| 3696 |
|
| 3697 | if (likely(src)) {
|
| 3698 | if (copy_mc_user_highpage(dst, src, addr, vma))
|
| 3699 | return -EHWPOISON;
|
| 3700 | return 0;
|
| 3701 | }
|
| 3702 |
|
| 3703 | |
| 3704 | * If the source page was a PFN mapping, we don't have
|
| 3705 | * a "struct page" for it. We do a best-effort copy by
|
| 3706 | * just copying from the original user address. If that
|
| 3707 | * fails, we just zero-fill it. Live with it.
|
| 3708 |
|
| 3709 | kaddr = kmap_local_page(dst);
|
| 3710 | pagefault_disable();
|
| 3711 | uaddr = (void __user *)(addr & PAGE_MASK);
|
| 3712 |
|
| 3713 | |
| 3714 | * On architectures with software "accessed" bits, we would
|
| 3715 | * take a double page fault, so mark it accessed here.
|
| 3716 |
|
| 3717 | vmf->pte = NULL;
|
| 3718 | if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
|
| 3719 | pte_t entry;
|
| 3720 |
|
| 3721 | vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
|
| 3722 | if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
|
| 3723 | |
| 3724 | * Other thread has already handled the fault
|
| 3725 | * and update local tlb only
|
| 3726 |
|
| 3727 | if (vmf->pte)
|
| 3728 | update_mmu_tlb(vma, addr, vmf->pte);
|
| 3729 | ret = -EAGAIN;
|
| 3730 | goto pte_unlock;
|
| 3731 | }
|
| 3732 |
|
| 3733 | entry = pte_mkyoung(vmf->orig_pte);
|
| 3734 | if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
|
| 3735 | update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
|
| 3736 | }
|
| 3737 |
|
| 3738 | |
| 3739 | * This really shouldn't fail, because the page is there
|
| 3740 | * in the page tables. But it might just be unreadable,
|
| 3741 | * in which case we just give up and fill the result with
|
| 3742 | * zeroes.
|
| 3743 |
|
| 3744 | if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
|
| 3745 | if (vmf->pte)
|
| 3746 | goto warn;
|
| 3747 |
|
| 3748 |
|
| 3749 | vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
|
| 3750 | if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
|
| 3751 |
|
| 3752 | if (vmf->pte)
|
| 3753 | update_mmu_tlb(vma, addr, vmf->pte);
|
| 3754 | ret = -EAGAIN;
|
| 3755 | goto pte_unlock;
|
| 3756 | }
|
| 3757 |
|
| 3758 | |
| 3759 | * The same page can be mapped back since last copy attempt.
|
| 3760 | * Try to copy again under PTL.
|
| 3761 |
|
| 3762 | if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
|
| 3763 | |
| 3764 | * Give a warn in case there can be some obscure
|
| 3765 | * use-case
|
| 3766 |
|
| 3767 | warn:
|
| 3768 | WARN_ON_ONCE(1);
|
| 3769 | clear_page(kaddr);
|
| 3770 | }
|
| 3771 | }
|
| 3772 |
|
| 3773 | ret = 0;
|
| 3774 |
|
| 3775 | pte_unlock:
|
| 3776 | if (vmf->pte)
|
| 3777 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 3778 | pagefault_enable();
|
| 3779 | kunmap_local(kaddr);
|
| 3780 | flush_dcache_page(dst);
|
| 3781 |
|
| 3782 | return ret;
|
| 3783 | }
|
| 3784 |
|
| 3785 | static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
|
| 3786 | {
|
| 3787 | struct file *vm_file = vma->vm_file;
|
| 3788 |
|
| 3789 | if (vm_file)
|
| 3790 | return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
|
| 3791 |
|
| 3792 | |
| 3793 | * Special mappings (e.g. VDSO) do not have any file so fake
|
| 3794 | * a default GFP_KERNEL for them.
|
| 3795 |
|
| 3796 | return GFP_KERNEL;
|
| 3797 | }
|
| 3798 |
|
| 3799 | |
| 3800 | * Notify the address space that the page is about to become writable so that
|
| 3801 | * it can prohibit this or wait for the page to get into an appropriate state.
|
| 3802 | *
|
| 3803 | * We do this without the lock held, so that it can sleep if it needs to.
|
| 3804 |
|
| 3805 | static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
|
| 3806 | {
|
| 3807 | vm_fault_t ret;
|
| 3808 | unsigned int old_flags = vmf->flags;
|
| 3809 |
|
| 3810 | vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
|
| 3811 |
|
| 3812 | if (vmf->vma->vm_file &&
|
| 3813 | IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
|
| 3814 | return VM_FAULT_SIGBUS;
|
| 3815 |
|
| 3816 | ret = vmf->vma->vm_ops->page_mkwrite(vmf);
|
| 3817 |
|
| 3818 | vmf->flags = old_flags;
|
| 3819 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
|
| 3820 | return ret;
|
| 3821 | if (unlikely(!(ret & VM_FAULT_LOCKED))) {
|
| 3822 | folio_lock(folio);
|
| 3823 | if (!folio->mapping) {
|
| 3824 | folio_unlock(folio);
|
| 3825 | return 0;
|
| 3826 | }
|
| 3827 | ret |= VM_FAULT_LOCKED;
|
| 3828 | } else
|
| 3829 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 3830 | return ret;
|
| 3831 | }
|
| 3832 |
|
| 3833 | |
| 3834 | * Handle dirtying of a page in shared file mapping on a write fault.
|
| 3835 | *
|
| 3836 | * The function expects the page to be locked and unlocks it.
|
| 3837 |
|
| 3838 | static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
|
| 3839 | {
|
| 3840 | struct vm_area_struct *vma = vmf->vma;
|
| 3841 | struct address_space *mapping;
|
| 3842 | struct folio *folio = page_folio(vmf->page);
|
| 3843 | bool dirtied;
|
| 3844 | bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
|
| 3845 |
|
| 3846 | dirtied = folio_mark_dirty(folio);
|
| 3847 | VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
|
| 3848 | |
| 3849 | * Take a local copy of the address_space - folio.mapping may be zeroed
|
| 3850 | * by truncate after folio_unlock(). The address_space itself remains
|
| 3851 | * pinned by vma->vm_file's reference. We rely on folio_unlock()'s
|
| 3852 | * release semantics to prevent the compiler from undoing this copying.
|
| 3853 |
|
| 3854 | mapping = folio_raw_mapping(folio);
|
| 3855 | folio_unlock(folio);
|
| 3856 |
|
| 3857 | if (!page_mkwrite)
|
| 3858 | file_update_time(vma->vm_file);
|
| 3859 |
|
| 3860 | |
| 3861 | * Throttle page dirtying rate down to writeback speed.
|
| 3862 | *
|
| 3863 | * mapping may be NULL here because some device drivers do not
|
| 3864 | * set page.mapping but still dirty their pages
|
| 3865 | *
|
| 3866 | * Drop the mmap_lock before waiting on IO, if we can. The file
|
| 3867 | * is pinning the mapping, as per above.
|
| 3868 |
|
| 3869 | if ((dirtied || page_mkwrite) && mapping) {
|
| 3870 | struct file *fpin;
|
| 3871 |
|
| 3872 | fpin = maybe_unlock_mmap_for_io(vmf, NULL);
|
| 3873 | balance_dirty_pages_ratelimited(mapping);
|
| 3874 | if (fpin) {
|
| 3875 | fput(fpin);
|
| 3876 | return VM_FAULT_COMPLETED;
|
| 3877 | }
|
| 3878 | }
|
| 3879 |
|
| 3880 | return 0;
|
| 3881 | }
|
| 3882 |
|
| 3883 | |
| 3884 | * Handle write page faults for pages that can be reused in the current vma
|
| 3885 | *
|
| 3886 | * This can happen either due to the mapping being with the VM_SHARED flag,
|
| 3887 | * or due to us being the last reference standing to the page. In either
|
| 3888 | * case, all we need to do here is to mark the page as writable and update
|
| 3889 | * any related book-keeping.
|
| 3890 |
|
| 3891 | static inline void wp_page_reuse(struct vm_fault *vmf, struct folio *folio)
|
| 3892 | __releases(vmf->ptl)
|
| 3893 | {
|
| 3894 | struct vm_area_struct *vma = vmf->vma;
|
| 3895 | pte_t entry;
|
| 3896 |
|
| 3897 | VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
|
| 3898 | VM_WARN_ON(is_zero_pfn(pte_pfn(vmf->orig_pte)));
|
| 3899 |
|
| 3900 | if (folio) {
|
| 3901 | VM_BUG_ON(folio_test_anon(folio) &&
|
| 3902 | !PageAnonExclusive(vmf->page));
|
| 3903 | |
| 3904 | * Clear the folio's cpupid information as the existing
|
| 3905 | * information potentially belongs to a now completely
|
| 3906 | * unrelated process.
|
| 3907 |
|
| 3908 | folio_xchg_last_cpupid(folio, (1 << LAST_CPUPID_SHIFT) - 1);
|
| 3909 | }
|
| 3910 |
|
| 3911 | flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
|
| 3912 | entry = pte_mkyoung(vmf->orig_pte);
|
| 3913 | entry = maybe_mkwrite(pte_mkdirty(entry), vma);
|
| 3914 | if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
|
| 3915 | update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
|
| 3916 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 3917 | count_vm_event(PGREUSE);
|
| 3918 | }
|
| 3919 |
|
| 3920 | |
| 3921 | * We could add a bitflag somewhere, but for now, we know that all
|
| 3922 | * vm_ops that have a ->map_pages have been audited and don't need
|
| 3923 | * the mmap_lock to be held.
|
| 3924 |
|
| 3925 | static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf)
|
| 3926 | {
|
| 3927 | struct vm_area_struct *vma = vmf->vma;
|
| 3928 |
|
| 3929 | if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK))
|
| 3930 | return 0;
|
| 3931 | vma_end_read(vma);
|
| 3932 | return VM_FAULT_RETRY;
|
| 3933 | }
|
| 3934 |
|
| 3935 | |
| 3936 | * __vmf_anon_prepare - Prepare to handle an anonymous fault.
|
| 3937 | * @vmf: The vm_fault descriptor passed from the fault handler.
|
| 3938 | *
|
| 3939 | * When preparing to insert an anonymous page into a VMA from a
|
| 3940 | * fault handler, call this function rather than anon_vma_prepare().
|
| 3941 | * If this vma does not already have an associated anon_vma and we are
|
| 3942 | * only protected by the per-VMA lock, the caller must retry with the
|
| 3943 | * mmap_lock held. __anon_vma_prepare() will look at adjacent VMAs to
|
| 3944 | * determine if this VMA can share its anon_vma, and that's not safe to
|
| 3945 | * do with only the per-VMA lock held for this VMA.
|
| 3946 | *
|
| 3947 | * Return: 0 if fault handling can proceed. Any other value should be
|
| 3948 | * returned to the caller.
|
| 3949 |
|
| 3950 | vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
|
| 3951 | {
|
| 3952 | struct vm_area_struct *vma = vmf->vma;
|
| 3953 | vm_fault_t ret = 0;
|
| 3954 |
|
| 3955 | if (likely(vma->anon_vma))
|
| 3956 | return 0;
|
| 3957 | if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
|
| 3958 | if (!mmap_read_trylock(vma->vm_mm))
|
| 3959 | return VM_FAULT_RETRY;
|
| 3960 | }
|
| 3961 | if (__anon_vma_prepare(vma))
|
| 3962 | ret = VM_FAULT_OOM;
|
| 3963 | if (vmf->flags & FAULT_FLAG_VMA_LOCK)
|
| 3964 | mmap_read_unlock(vma->vm_mm);
|
| 3965 | return ret;
|
| 3966 | }
|
| 3967 |
|
| 3968 | |
| 3969 | * Handle the case of a page which we actually need to copy to a new page,
|
| 3970 | * either due to COW or unsharing.
|
| 3971 | *
|
| 3972 | * Called with either the VMA lock or the mmap_lock held (see FAULT_FLAG_VMA_LOCK)
|
| 3973 | * and the old page referenced, but without the ptl held.
|
| 3974 | *
|
| 3975 | * High level logic flow:
|
| 3976 | *
|
| 3977 | * - Allocate a page, copy the content of the old page to the new one.
|
| 3978 | * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
|
| 3979 | * - Take the PTL. If the pte changed, bail out and release the allocated page
|
| 3980 | * - If the pte is still the way we remember it, update the page table and all
|
| 3981 | * relevant references. This includes dropping the reference the page-table
|
| 3982 | * held to the old page, as well as updating the rmap.
|
| 3983 | * - In any case, unlock the PTL and drop the reference we took to the old page.
|
| 3984 |
|
| 3985 | static vm_fault_t wp_page_copy(struct vm_fault *vmf)
|
| 3986 | {
|
| 3987 | const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
|
| 3988 | struct vm_area_struct *vma = vmf->vma;
|
| 3989 | struct mm_struct *mm = vma->vm_mm;
|
| 3990 | struct folio *old_folio = NULL;
|
| 3991 | struct folio *new_folio = NULL;
|
| 3992 | pte_t entry;
|
| 3993 | int page_copied = 0;
|
| 3994 | struct mmu_notifier_range range;
|
| 3995 | vm_fault_t ret;
|
| 3996 | bool pfn_is_zero;
|
| 3997 |
|
| 3998 | delayacct_wpcopy_start();
|
| 3999 |
|
| 4000 | if (vmf->page)
|
| 4001 | old_folio = page_folio(vmf->page);
|
| 4002 | ret = vmf_anon_prepare(vmf);
|
| 4003 | if (unlikely(ret))
|
| 4004 | goto out;
|
| 4005 |
|
| 4006 | pfn_is_zero = is_zero_pfn(pte_pfn(vmf->orig_pte));
|
| 4007 | new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero);
|
| 4008 | if (!new_folio)
|
| 4009 | goto oom;
|
| 4010 |
|
| 4011 | if (!pfn_is_zero) {
|
| 4012 | int err;
|
| 4013 |
|
| 4014 | err = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
|
| 4015 | if (err) {
|
| 4016 | |
| 4017 | * COW failed, if the fault was solved by other,
|
| 4018 | * it's fine. If not, userspace would re-fault on
|
| 4019 | * the same address and we will handle the fault
|
| 4020 | * from the second attempt.
|
| 4021 | * The -EHWPOISON case will not be retried.
|
| 4022 |
|
| 4023 | folio_put(new_folio);
|
| 4024 | if (old_folio)
|
| 4025 | folio_put(old_folio);
|
| 4026 |
|
| 4027 | delayacct_wpcopy_end();
|
| 4028 | return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
|
| 4029 | }
|
| 4030 | kmsan_copy_page_meta(&new_folio->page, vmf->page);
|
| 4031 | }
|
| 4032 |
|
| 4033 | __folio_mark_uptodate(new_folio);
|
| 4034 |
|
| 4035 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
|
| 4036 | vmf->address & PAGE_MASK,
|
| 4037 | (vmf->address & PAGE_MASK) + PAGE_SIZE);
|
| 4038 | mmu_notifier_invalidate_range_start(&range);
|
| 4039 |
|
| 4040 | |
| 4041 | * Re-check the pte - we dropped the lock
|
| 4042 |
|
| 4043 | vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
|
| 4044 | if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
|
| 4045 | if (old_folio) {
|
| 4046 | if (!folio_test_anon(old_folio)) {
|
| 4047 | dec_mm_counter(mm, mm_counter_file(old_folio));
|
| 4048 | inc_mm_counter(mm, MM_ANONPAGES);
|
| 4049 | }
|
| 4050 | } else {
|
| 4051 | ksm_might_unmap_zero_page(mm, vmf->orig_pte);
|
| 4052 | inc_mm_counter(mm, MM_ANONPAGES);
|
| 4053 | }
|
| 4054 | flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
|
| 4055 | entry = folio_mk_pte(new_folio, vma->vm_page_prot);
|
| 4056 | entry = pte_sw_mkyoung(entry);
|
| 4057 | if (unlikely(unshare)) {
|
| 4058 | if (pte_soft_dirty(vmf->orig_pte))
|
| 4059 | entry = pte_mksoft_dirty(entry);
|
| 4060 | if (pte_uffd(vmf->orig_pte))
|
| 4061 | entry = pte_mkuffd(entry);
|
| 4062 | } else {
|
| 4063 | entry = maybe_mkwrite(pte_mkdirty(entry), vma);
|
| 4064 | }
|
| 4065 |
|
| 4066 | |
| 4067 | * Clear the pte entry and flush it first, before updating the
|
| 4068 | * pte with the new entry, to keep TLBs on different CPUs in
|
| 4069 | * sync. This code used to set the new PTE then flush TLBs, but
|
| 4070 | * that left a window where the new PTE could be loaded into
|
| 4071 | * some TLBs while the old PTE remains in others.
|
| 4072 |
|
| 4073 | ptep_clear_flush(vma, vmf->address, vmf->pte);
|
| 4074 | folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE);
|
| 4075 | folio_add_lru_vma(new_folio, vma);
|
| 4076 | BUG_ON(unshare && pte_write(entry));
|
| 4077 | set_pte_at(mm, vmf->address, vmf->pte, entry);
|
| 4078 | update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
|
| 4079 | if (old_folio) {
|
| 4080 | |
| 4081 | * Only after switching the pte to the new page may
|
| 4082 | * we remove the mapcount here. Otherwise another
|
| 4083 | * process may come and find the rmap count decremented
|
| 4084 | * before the pte is switched to the new page, and
|
| 4085 | * "reuse" the old page writing into it while our pte
|
| 4086 | * here still points into it and can be read by other
|
| 4087 | * threads.
|
| 4088 | *
|
| 4089 | * The critical issue is to order this
|
| 4090 | * folio_remove_rmap_pte() with the ptp_clear_flush
|
| 4091 | * above. Those stores are ordered by (if nothing else,)
|
| 4092 | * the barrier present in the atomic_add_negative
|
| 4093 | * in folio_remove_rmap_pte();
|
| 4094 | *
|
| 4095 | * Then the TLB flush in ptep_clear_flush ensures that
|
| 4096 | * no process can access the old page before the
|
| 4097 | * decremented mapcount is visible. And the old page
|
| 4098 | * cannot be reused until after the decremented
|
| 4099 | * mapcount is visible. So transitively, TLBs to
|
| 4100 | * old page will be flushed before it can be reused.
|
| 4101 |
|
| 4102 | folio_remove_rmap_pte(old_folio, vmf->page, vma);
|
| 4103 | }
|
| 4104 |
|
| 4105 |
|
| 4106 | new_folio = old_folio;
|
| 4107 | page_copied = 1;
|
| 4108 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4109 | } else if (vmf->pte) {
|
| 4110 | update_mmu_tlb(vma, vmf->address, vmf->pte);
|
| 4111 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4112 | }
|
| 4113 |
|
| 4114 | mmu_notifier_invalidate_range_end(&range);
|
| 4115 |
|
| 4116 | if (new_folio)
|
| 4117 | folio_put(new_folio);
|
| 4118 | if (old_folio) {
|
| 4119 | if (page_copied)
|
| 4120 | free_swap_cache(old_folio);
|
| 4121 | folio_put(old_folio);
|
| 4122 | }
|
| 4123 |
|
| 4124 | delayacct_wpcopy_end();
|
| 4125 | return 0;
|
| 4126 | oom:
|
| 4127 | ret = VM_FAULT_OOM;
|
| 4128 | out:
|
| 4129 | if (old_folio)
|
| 4130 | folio_put(old_folio);
|
| 4131 |
|
| 4132 | delayacct_wpcopy_end();
|
| 4133 | return ret;
|
| 4134 | }
|
| 4135 |
|
| 4136 | |
| 4137 | * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
|
| 4138 | * writeable once the page is prepared
|
| 4139 | *
|
| 4140 | * @vmf: structure describing the fault
|
| 4141 | * @folio: the folio of vmf->page
|
| 4142 | *
|
| 4143 | * This function handles all that is needed to finish a write page fault in a
|
| 4144 | * shared mapping due to PTE being read-only once the mapped page is prepared.
|
| 4145 | * It handles locking of PTE and modifying it.
|
| 4146 | *
|
| 4147 | * The function expects the page to be locked or other protection against
|
| 4148 | * concurrent faults / writeback (such as DAX radix tree locks).
|
| 4149 | *
|
| 4150 | * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
|
| 4151 | * we acquired PTE lock.
|
| 4152 |
|
| 4153 | static vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf, struct folio *folio)
|
| 4154 | {
|
| 4155 | WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
|
| 4156 | vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
|
| 4157 | &vmf->ptl);
|
| 4158 | if (!vmf->pte)
|
| 4159 | return VM_FAULT_NOPAGE;
|
| 4160 | |
| 4161 | * We might have raced with another page fault while we released the
|
| 4162 | * pte_offset_map_lock.
|
| 4163 |
|
| 4164 | if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
|
| 4165 | update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
|
| 4166 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4167 | return VM_FAULT_NOPAGE;
|
| 4168 | }
|
| 4169 | wp_page_reuse(vmf, folio);
|
| 4170 | return 0;
|
| 4171 | }
|
| 4172 |
|
| 4173 | |
| 4174 | * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
|
| 4175 | * mapping
|
| 4176 |
|
| 4177 | static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
|
| 4178 | {
|
| 4179 | struct vm_area_struct *vma = vmf->vma;
|
| 4180 |
|
| 4181 | if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
|
| 4182 | vm_fault_t ret;
|
| 4183 |
|
| 4184 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4185 | ret = vmf_can_call_fault(vmf);
|
| 4186 | if (ret)
|
| 4187 | return ret;
|
| 4188 |
|
| 4189 | vmf->flags |= FAULT_FLAG_MKWRITE;
|
| 4190 | ret = vma->vm_ops->pfn_mkwrite(vmf);
|
| 4191 | if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
|
| 4192 | return ret;
|
| 4193 | return finish_mkwrite_fault(vmf, NULL);
|
| 4194 | }
|
| 4195 | wp_page_reuse(vmf, NULL);
|
| 4196 | return 0;
|
| 4197 | }
|
| 4198 |
|
| 4199 | static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
|
| 4200 | __releases(vmf->ptl)
|
| 4201 | {
|
| 4202 | struct vm_area_struct *vma = vmf->vma;
|
| 4203 | vm_fault_t ret = 0;
|
| 4204 |
|
| 4205 | folio_get(folio);
|
| 4206 |
|
| 4207 | if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
|
| 4208 | vm_fault_t tmp;
|
| 4209 |
|
| 4210 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4211 | tmp = vmf_can_call_fault(vmf);
|
| 4212 | if (tmp) {
|
| 4213 | folio_put(folio);
|
| 4214 | return tmp;
|
| 4215 | }
|
| 4216 |
|
| 4217 | tmp = do_page_mkwrite(vmf, folio);
|
| 4218 | if (unlikely(!tmp || (tmp &
|
| 4219 | (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
|
| 4220 | folio_put(folio);
|
| 4221 | return tmp;
|
| 4222 | }
|
| 4223 | tmp = finish_mkwrite_fault(vmf, folio);
|
| 4224 | if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
|
| 4225 | folio_unlock(folio);
|
| 4226 | folio_put(folio);
|
| 4227 | return tmp;
|
| 4228 | }
|
| 4229 | } else {
|
| 4230 | wp_page_reuse(vmf, folio);
|
| 4231 | folio_lock(folio);
|
| 4232 | }
|
| 4233 | ret |= fault_dirty_shared_page(vmf);
|
| 4234 | folio_put(folio);
|
| 4235 |
|
| 4236 | return ret;
|
| 4237 | }
|
| 4238 |
|
| 4239 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 4240 | static bool __wp_can_reuse_large_anon_folio(struct folio *folio,
|
| 4241 | struct vm_area_struct *vma)
|
| 4242 | {
|
| 4243 | bool exclusive = false;
|
| 4244 |
|
| 4245 |
|
| 4246 | if (folio_large_mapcount(folio) <= 1)
|
| 4247 | return false;
|
| 4248 |
|
| 4249 | |
| 4250 | * The assumption for anonymous folios is that each page can only get
|
| 4251 | * mapped once into each MM. The only exception are KSM folios, which
|
| 4252 | * are always small.
|
| 4253 | *
|
| 4254 | * Each taken mapcount must be paired with exactly one taken reference,
|
| 4255 | * whereby the refcount must be incremented before the mapcount when
|
| 4256 | * mapping a page, and the refcount must be decremented after the
|
| 4257 | * mapcount when unmapping a page.
|
| 4258 | *
|
| 4259 | * If all folio references are from mappings, and all mappings are in
|
| 4260 | * the page tables of this MM, then this folio is exclusive to this MM.
|
| 4261 |
|
| 4262 | if (test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids))
|
| 4263 | return false;
|
| 4264 |
|
| 4265 | VM_WARN_ON_ONCE(folio_test_ksm(folio));
|
| 4266 |
|
| 4267 | if (unlikely(folio_test_swapcache(folio))) {
|
| 4268 | |
| 4269 | * Note: freeing up the swapcache will fail if some PTEs are
|
| 4270 | * still swap entries.
|
| 4271 |
|
| 4272 | if (!folio_trylock(folio))
|
| 4273 | return false;
|
| 4274 | folio_free_swap(folio);
|
| 4275 | folio_unlock(folio);
|
| 4276 | }
|
| 4277 |
|
| 4278 | if (folio_large_mapcount(folio) != folio_ref_count(folio))
|
| 4279 | return false;
|
| 4280 |
|
| 4281 |
|
| 4282 | folio_lock_large_mapcount(folio);
|
| 4283 | VM_WARN_ON_ONCE_FOLIO(folio_large_mapcount(folio) > folio_ref_count(folio), folio);
|
| 4284 |
|
| 4285 | if (test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids))
|
| 4286 | goto unlock;
|
| 4287 | if (folio_large_mapcount(folio) != folio_ref_count(folio))
|
| 4288 | goto unlock;
|
| 4289 |
|
| 4290 | VM_WARN_ON_ONCE_FOLIO(folio_large_mapcount(folio) > folio_nr_pages(folio), folio);
|
| 4291 | VM_WARN_ON_ONCE_FOLIO(folio_entire_mapcount(folio), folio);
|
| 4292 | VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != vma->vm_mm->mm_id &&
|
| 4293 | folio_mm_id(folio, 1) != vma->vm_mm->mm_id);
|
| 4294 |
|
| 4295 | |
| 4296 | * Do we need the folio lock? Likely not. If there would have been
|
| 4297 | * references from page migration/swapout, we would have detected
|
| 4298 | * an additional folio reference and never ended up here.
|
| 4299 |
|
| 4300 | exclusive = true;
|
| 4301 | unlock:
|
| 4302 | folio_unlock_large_mapcount(folio);
|
| 4303 | return exclusive;
|
| 4304 | }
|
| 4305 | #else
|
| 4306 | static bool __wp_can_reuse_large_anon_folio(struct folio *folio,
|
| 4307 | struct vm_area_struct *vma)
|
| 4308 | {
|
| 4309 | BUILD_BUG();
|
| 4310 | }
|
| 4311 | #endif
|
| 4312 |
|
| 4313 | static bool wp_can_reuse_anon_folio(struct folio *folio,
|
| 4314 | struct vm_area_struct *vma)
|
| 4315 | {
|
| 4316 | const bool maybe_in_lru_cache = !folio_test_lru(folio);
|
| 4317 | const bool in_swapcache = folio_test_swapcache(folio);
|
| 4318 |
|
| 4319 | if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio))
|
| 4320 | return __wp_can_reuse_large_anon_folio(folio, vma);
|
| 4321 |
|
| 4322 | |
| 4323 | * We have to verify under folio lock: these early checks are
|
| 4324 | * just an optimization to avoid locking the folio and freeing
|
| 4325 | * the swapcache if there is little hope that we can reuse.
|
| 4326 | *
|
| 4327 | * KSM doesn't necessarily raise the folio refcount.
|
| 4328 |
|
| 4329 | if (folio_test_ksm(folio) ||
|
| 4330 | folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache)
|
| 4331 | return false;
|
| 4332 | if (maybe_in_lru_cache)
|
| 4333 | |
| 4334 | * We cannot easily detect+handle references from
|
| 4335 | * remote LRU caches or references to LRU folios.
|
| 4336 |
|
| 4337 | lru_add_drain();
|
| 4338 | if (folio_ref_count(folio) > 1 + in_swapcache)
|
| 4339 | return false;
|
| 4340 | if (!folio_trylock(folio))
|
| 4341 | return false;
|
| 4342 | if (folio_test_swapcache(folio))
|
| 4343 | folio_free_swap(folio);
|
| 4344 | if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
|
| 4345 | folio_unlock(folio);
|
| 4346 | return false;
|
| 4347 | }
|
| 4348 | |
| 4349 | * Ok, we've got the only folio reference from our mapping
|
| 4350 | * and the folio is locked, it's dark out, and we're wearing
|
| 4351 | * sunglasses. Hit it.
|
| 4352 |
|
| 4353 | folio_move_anon_rmap(folio, vma);
|
| 4354 | folio_unlock(folio);
|
| 4355 | return true;
|
| 4356 | }
|
| 4357 |
|
| 4358 | |
| 4359 | * This routine handles present pages, when
|
| 4360 | * * users try to write to a shared page (FAULT_FLAG_WRITE)
|
| 4361 | * * GUP wants to take a R/O pin on a possibly shared anonymous page
|
| 4362 | * (FAULT_FLAG_UNSHARE)
|
| 4363 | *
|
| 4364 | * It is done by copying the page to a new address and decrementing the
|
| 4365 | * shared-page counter for the old page.
|
| 4366 | *
|
| 4367 | * Note that this routine assumes that the protection checks have been
|
| 4368 | * done by the caller (the low-level page fault routine in most cases).
|
| 4369 | * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
|
| 4370 | * done any necessary COW.
|
| 4371 | *
|
| 4372 | * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
|
| 4373 | * though the page will change only once the write actually happens. This
|
| 4374 | * avoids a few races, and potentially makes it more efficient.
|
| 4375 | *
|
| 4376 | * We enter with either the VMA lock or the mmap_lock held (see
|
| 4377 | * FAULT_FLAG_VMA_LOCK) and pte both mapped and locked. We return with
|
| 4378 | * the same lock still held, but pte unmapped and unlocked.
|
| 4379 |
|
| 4380 | static vm_fault_t do_wp_page(struct vm_fault *vmf)
|
| 4381 | __releases(vmf->ptl)
|
| 4382 | {
|
| 4383 | const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
|
| 4384 | struct vm_area_struct *vma = vmf->vma;
|
| 4385 | struct folio *folio = NULL;
|
| 4386 | pte_t pte;
|
| 4387 |
|
| 4388 | if (likely(!unshare)) {
|
| 4389 | if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
|
| 4390 | if (!userfaultfd_wp_async(vma)) {
|
| 4391 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4392 | return handle_userfault(vmf, VM_UFFD_WP);
|
| 4393 | }
|
| 4394 |
|
| 4395 | |
| 4396 | * Nothing needed (cache flush, TLB invalidations,
|
| 4397 | * etc.) because we're only removing the uffd-wp bit,
|
| 4398 | * which is completely invisible to the user.
|
| 4399 |
|
| 4400 | pte = pte_clear_uffd(ptep_get(vmf->pte));
|
| 4401 |
|
| 4402 | set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
|
| 4403 | |
| 4404 | * Update this to be prepared for following up CoW
|
| 4405 | * handling
|
| 4406 |
|
| 4407 | vmf->orig_pte = pte;
|
| 4408 | }
|
| 4409 |
|
| 4410 | |
| 4411 | * Userfaultfd write-protect can defer flushes. Ensure the TLB
|
| 4412 | * is flushed in this case before copying.
|
| 4413 |
|
| 4414 | if (unlikely(userfaultfd_wp(vmf->vma) &&
|
| 4415 | mm_tlb_flush_pending(vmf->vma->vm_mm)))
|
| 4416 | flush_tlb_page(vmf->vma, vmf->address);
|
| 4417 | }
|
| 4418 |
|
| 4419 | vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
|
| 4420 |
|
| 4421 | if (vmf->page)
|
| 4422 | folio = page_folio(vmf->page);
|
| 4423 |
|
| 4424 | |
| 4425 | * Shared mapping: we are guaranteed to have VM_WRITE and
|
| 4426 | * FAULT_FLAG_WRITE set at this point.
|
| 4427 |
|
| 4428 | if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
|
| 4429 | |
| 4430 | * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
|
| 4431 | * VM_PFNMAP VMA. FS DAX also wants ops->pfn_mkwrite called.
|
| 4432 | *
|
| 4433 | * We should not cow pages in a shared writeable mapping.
|
| 4434 | * Just mark the pages writable and/or call ops->pfn_mkwrite.
|
| 4435 |
|
| 4436 | if (!vmf->page || is_fsdax_page(vmf->page)) {
|
| 4437 | vmf->page = NULL;
|
| 4438 | return wp_pfn_shared(vmf);
|
| 4439 | }
|
| 4440 | return wp_page_shared(vmf, folio);
|
| 4441 | }
|
| 4442 |
|
| 4443 | |
| 4444 | * Private mapping: create an exclusive anonymous page copy if reuse
|
| 4445 | * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
|
| 4446 | *
|
| 4447 | * If we encounter a page that is marked exclusive, we must reuse
|
| 4448 | * the page without further checks.
|
| 4449 |
|
| 4450 | if (folio && folio_test_anon(folio) &&
|
| 4451 | (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) {
|
| 4452 | if (!PageAnonExclusive(vmf->page))
|
| 4453 | SetPageAnonExclusive(vmf->page);
|
| 4454 | if (unlikely(unshare)) {
|
| 4455 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4456 | return 0;
|
| 4457 | }
|
| 4458 | wp_page_reuse(vmf, folio);
|
| 4459 | return 0;
|
| 4460 | }
|
| 4461 | |
| 4462 | * Ok, we need to copy. Oh, well..
|
| 4463 |
|
| 4464 | if (folio)
|
| 4465 | folio_get(folio);
|
| 4466 |
|
| 4467 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4468 | #ifdef CONFIG_KSM
|
| 4469 | if (folio && folio_test_ksm(folio))
|
| 4470 | count_vm_event(COW_KSM);
|
| 4471 | #endif
|
| 4472 | return wp_page_copy(vmf);
|
| 4473 | }
|
| 4474 |
|
| 4475 | static inline void unmap_mapping_range_tree(struct address_space *mapping,
|
| 4476 | pgoff_t first_index,
|
| 4477 | pgoff_t last_index,
|
| 4478 | struct zap_details *details)
|
| 4479 | {
|
| 4480 | struct vm_area_struct *vma;
|
| 4481 | struct mmu_gather tlb;
|
| 4482 |
|
| 4483 | mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) {
|
| 4484 | const pgoff_t start_idx = max(first_index, vma_start_pgoff(vma));
|
| 4485 | const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1;
|
| 4486 | const pgoff_t offset = start_idx - vma_start_pgoff(vma);
|
| 4487 | const unsigned long offset_bytes = offset << PAGE_SHIFT;
|
| 4488 | const unsigned long start = vma->vm_start + offset_bytes;
|
| 4489 | const unsigned long size = (end_idx - start_idx) << PAGE_SHIFT;
|
| 4490 |
|
| 4491 | tlb_gather_mmu(&tlb, vma->vm_mm);
|
| 4492 | zap_vma_range_batched(&tlb, vma, start, size, details);
|
| 4493 | tlb_finish_mmu(&tlb);
|
| 4494 | }
|
| 4495 | }
|
| 4496 |
|
| 4497 | |
| 4498 | * unmap_mapping_folio() - Unmap single folio from processes.
|
| 4499 | * @folio: The locked folio to be unmapped.
|
| 4500 | *
|
| 4501 | * Unmap this folio from any userspace process which still has it mmaped.
|
| 4502 | * Typically, for efficiency, the range of nearby pages has already been
|
| 4503 | * unmapped by unmap_mapping_pages() or unmap_mapping_range(). But once
|
| 4504 | * truncation or invalidation holds the lock on a folio, it may find that
|
| 4505 | * the page has been remapped again: and then uses unmap_mapping_folio()
|
| 4506 | * to unmap it finally.
|
| 4507 |
|
| 4508 | void unmap_mapping_folio(struct folio *folio)
|
| 4509 | {
|
| 4510 | struct address_space *mapping = folio->mapping;
|
| 4511 | struct zap_details details = { };
|
| 4512 | pgoff_t first_index;
|
| 4513 | pgoff_t last_index;
|
| 4514 |
|
| 4515 | VM_BUG_ON(!folio_test_locked(folio));
|
| 4516 |
|
| 4517 | first_index = folio->index;
|
| 4518 | last_index = folio_next_index(folio) - 1;
|
| 4519 |
|
| 4520 | details.skip_cows = true;
|
| 4521 | details.single_folio = folio;
|
| 4522 | details.zap_flags = ZAP_FLAG_DROP_MARKER;
|
| 4523 |
|
| 4524 | i_mmap_lock_read(mapping);
|
| 4525 | if (unlikely(mapping_mapped(mapping)))
|
| 4526 | unmap_mapping_range_tree(mapping, first_index,
|
| 4527 | last_index, &details);
|
| 4528 | i_mmap_unlock_read(mapping);
|
| 4529 | }
|
| 4530 |
|
| 4531 | |
| 4532 | * unmap_mapping_pages() - Unmap pages from processes.
|
| 4533 | * @mapping: The address space containing pages to be unmapped.
|
| 4534 | * @start: Index of first page to be unmapped.
|
| 4535 | * @nr: Number of pages to be unmapped. 0 to unmap to end of file.
|
| 4536 | * @even_cows: Whether to unmap even private COWed pages.
|
| 4537 | *
|
| 4538 | * Unmap the pages in this address space from any userspace process which
|
| 4539 | * has them mmaped. Generally, you want to remove COWed pages as well when
|
| 4540 | * a file is being truncated, but not when invalidating pages from the page
|
| 4541 | * cache.
|
| 4542 |
|
| 4543 | void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
|
| 4544 | pgoff_t nr, bool even_cows)
|
| 4545 | {
|
| 4546 | struct zap_details details = { };
|
| 4547 | pgoff_t first_index = start;
|
| 4548 | pgoff_t last_index = start + nr - 1;
|
| 4549 |
|
| 4550 | details.skip_cows = !even_cows;
|
| 4551 | if (last_index < first_index)
|
| 4552 | last_index = ULONG_MAX;
|
| 4553 |
|
| 4554 | i_mmap_lock_read(mapping);
|
| 4555 | if (unlikely(mapping_mapped(mapping)))
|
| 4556 | unmap_mapping_range_tree(mapping, first_index,
|
| 4557 | last_index, &details);
|
| 4558 | i_mmap_unlock_read(mapping);
|
| 4559 | }
|
| 4560 | EXPORT_SYMBOL_GPL(unmap_mapping_pages);
|
| 4561 |
|
| 4562 | |
| 4563 | * unmap_mapping_range - unmap the portion of all mmaps in the specified
|
| 4564 | * address_space corresponding to the specified byte range in the underlying
|
| 4565 | * file.
|
| 4566 | *
|
| 4567 | * @mapping: the address space containing mmaps to be unmapped.
|
| 4568 | * @holebegin: byte in first page to unmap, relative to the start of
|
| 4569 | * the underlying file. This will be rounded down to a PAGE_SIZE
|
| 4570 | * boundary. Note that this is different from truncate_pagecache(), which
|
| 4571 | * must keep the partial page. In contrast, we must get rid of
|
| 4572 | * partial pages.
|
| 4573 | * @holelen: size of prospective hole in bytes. This will be rounded
|
| 4574 | * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
|
| 4575 | * end of the file.
|
| 4576 | * @even_cows: 1 when truncating a file, unmap even private COWed pages;
|
| 4577 | * but 0 when invalidating pagecache, don't throw away private data.
|
| 4578 |
|
| 4579 | void unmap_mapping_range(struct address_space *mapping,
|
| 4580 | loff_t const holebegin, loff_t const holelen, int even_cows)
|
| 4581 | {
|
| 4582 | pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
|
| 4583 | pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
|
| 4584 |
|
| 4585 |
|
| 4586 | if (sizeof(holelen) > sizeof(hlen)) {
|
| 4587 | long long holeend =
|
| 4588 | (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
|
| 4589 | if (holeend & ~(long long)ULONG_MAX)
|
| 4590 | hlen = ULONG_MAX - hba + 1;
|
| 4591 | }
|
| 4592 |
|
| 4593 | unmap_mapping_pages(mapping, hba, hlen, even_cows);
|
| 4594 | }
|
| 4595 | EXPORT_SYMBOL(unmap_mapping_range);
|
| 4596 |
|
| 4597 | |
| 4598 | * Restore a potential device exclusive pte to a working pte entry
|
| 4599 |
|
| 4600 | static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
|
| 4601 | {
|
| 4602 | struct folio *folio = page_folio(vmf->page);
|
| 4603 | struct vm_area_struct *vma = vmf->vma;
|
| 4604 | struct mmu_notifier_range range;
|
| 4605 | vm_fault_t ret;
|
| 4606 |
|
| 4607 | |
| 4608 | * We need a reference to lock the folio because we don't hold
|
| 4609 | * the PTL so a racing thread can remove the device-exclusive
|
| 4610 | * entry and unmap it. If the folio is free the entry must
|
| 4611 | * have been removed already. If it happens to have already
|
| 4612 | * been re-allocated after being freed all we do is lock and
|
| 4613 | * unlock it.
|
| 4614 |
|
| 4615 | if (!folio_try_get(folio))
|
| 4616 | return 0;
|
| 4617 |
|
| 4618 | ret = folio_lock_or_retry(folio, vmf);
|
| 4619 | if (ret) {
|
| 4620 | folio_put(folio);
|
| 4621 | return ret;
|
| 4622 | }
|
| 4623 | mmu_notifier_range_init_owner(&range, MMU_NOTIFY_CLEAR, 0,
|
| 4624 | vma->vm_mm, vmf->address & PAGE_MASK,
|
| 4625 | (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
|
| 4626 | mmu_notifier_invalidate_range_start(&range);
|
| 4627 |
|
| 4628 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
|
| 4629 | &vmf->ptl);
|
| 4630 | if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
|
| 4631 | restore_exclusive_pte(vma, folio, vmf->page, vmf->address,
|
| 4632 | vmf->pte, vmf->orig_pte);
|
| 4633 |
|
| 4634 | if (vmf->pte)
|
| 4635 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4636 | folio_unlock(folio);
|
| 4637 | folio_put(folio);
|
| 4638 |
|
| 4639 | mmu_notifier_invalidate_range_end(&range);
|
| 4640 | return 0;
|
| 4641 | }
|
| 4642 |
|
| 4643 | |
| 4644 | * Check if we should call folio_free_swap to free the swap cache.
|
| 4645 | * folio_free_swap only frees the swap cache to release the slot if swap
|
| 4646 | * count is zero, so we don't need to check the swap count here.
|
| 4647 |
|
| 4648 | static inline bool should_try_to_free_swap(struct swap_info_struct *si,
|
| 4649 | struct folio *folio,
|
| 4650 | struct vm_area_struct *vma,
|
| 4651 | bool exclusive,
|
| 4652 | unsigned int fault_flags)
|
| 4653 | {
|
| 4654 | if (!folio_test_swapcache(folio))
|
| 4655 | return false;
|
| 4656 | |
| 4657 | * Always try to free swap cache for SWP_SYNCHRONOUS_IO devices. Swap
|
| 4658 | * cache can help save some IO or memory overhead, but these devices
|
| 4659 | * are fast, and meanwhile, swap cache pinning the slot deferring the
|
| 4660 | * release of metadata or fragmentation is a more critical issue.
|
| 4661 |
|
| 4662 | if (data_race(si->flags & SWP_SYNCHRONOUS_IO))
|
| 4663 | return true;
|
| 4664 | if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
|
| 4665 | folio_test_mlocked(folio))
|
| 4666 | return true;
|
| 4667 |
|
| 4668 | |
| 4669 | * Free the swapcache only if we are the exclusive user and
|
| 4670 | * this is a write fault.
|
| 4671 |
|
| 4672 | return (fault_flags & FAULT_FLAG_WRITE) && exclusive;
|
| 4673 | }
|
| 4674 |
|
| 4675 | static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
|
| 4676 | {
|
| 4677 | vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
|
| 4678 | vmf->address, &vmf->ptl);
|
| 4679 | if (!vmf->pte)
|
| 4680 | return 0;
|
| 4681 | |
| 4682 | * Be careful so that we will only recover a special uffd-wp pte into a
|
| 4683 | * none pte. Otherwise it means the pte could have changed, so retry.
|
| 4684 | *
|
| 4685 | * This should also cover the case where e.g. the pte changed
|
| 4686 | * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
|
| 4687 | * So pte_is_marker() check is not enough to safely drop the pte.
|
| 4688 |
|
| 4689 | if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
|
| 4690 | pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
|
| 4691 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4692 | return 0;
|
| 4693 | }
|
| 4694 |
|
| 4695 | static vm_fault_t do_pte_missing(struct vm_fault *vmf)
|
| 4696 | {
|
| 4697 | if (vma_is_anonymous(vmf->vma))
|
| 4698 | return do_anonymous_page(vmf);
|
| 4699 | else
|
| 4700 | return do_fault(vmf);
|
| 4701 | }
|
| 4702 |
|
| 4703 | |
| 4704 | * This is actually a page-missing access, but with uffd-wp special pte
|
| 4705 | * installed. It means this pte was wr-protected before being unmapped.
|
| 4706 |
|
| 4707 | static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
|
| 4708 | {
|
| 4709 | |
| 4710 | * Just in case there're leftover special ptes even after the region
|
| 4711 | * got unregistered - we can simply clear them.
|
| 4712 |
|
| 4713 | if (unlikely(!userfaultfd_wp(vmf->vma)))
|
| 4714 | return pte_marker_clear(vmf);
|
| 4715 |
|
| 4716 | return do_pte_missing(vmf);
|
| 4717 | }
|
| 4718 |
|
| 4719 | static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
|
| 4720 | {
|
| 4721 | const softleaf_t entry = softleaf_from_pte(vmf->orig_pte);
|
| 4722 | const pte_marker marker = softleaf_to_marker(entry);
|
| 4723 |
|
| 4724 | |
| 4725 | * PTE markers should never be empty. If anything weird happened,
|
| 4726 | * the best thing to do is to kill the process along with its mm.
|
| 4727 |
|
| 4728 | if (WARN_ON_ONCE(!marker))
|
| 4729 | return VM_FAULT_SIGBUS;
|
| 4730 |
|
| 4731 |
|
| 4732 | if (marker & PTE_MARKER_POISONED)
|
| 4733 | return VM_FAULT_HWPOISON;
|
| 4734 |
|
| 4735 |
|
| 4736 | if (marker & PTE_MARKER_GUARD)
|
| 4737 | return VM_FAULT_SIGSEGV;
|
| 4738 |
|
| 4739 | if (softleaf_is_uffd_wp_marker(entry))
|
| 4740 | return pte_marker_handle_uffd_wp(vmf);
|
| 4741 |
|
| 4742 |
|
| 4743 | return VM_FAULT_SIGBUS;
|
| 4744 | }
|
| 4745 |
|
| 4746 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 4747 | |
| 4748 | * Check if the PTEs within a range are contiguous swap entries.
|
| 4749 |
|
| 4750 | static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
|
| 4751 | {
|
| 4752 | unsigned long addr;
|
| 4753 | int idx;
|
| 4754 | pte_t pte;
|
| 4755 |
|
| 4756 | addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
|
| 4757 | idx = (vmf->address - addr) / PAGE_SIZE;
|
| 4758 | pte = ptep_get(ptep);
|
| 4759 |
|
| 4760 | if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
|
| 4761 | return false;
|
| 4762 | |
| 4763 | * swap_read_folio() can't handle the case a large folio is hybridly
|
| 4764 | * from different backends. And they are likely corner cases. Similar
|
| 4765 | * things might be added once zswap support large folios.
|
| 4766 |
|
| 4767 | if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
|
| 4768 | return false;
|
| 4769 | return true;
|
| 4770 | }
|
| 4771 |
|
| 4772 | static inline unsigned long thp_swap_suitable_orders(pgoff_t swp_offset,
|
| 4773 | unsigned long addr,
|
| 4774 | unsigned long orders)
|
| 4775 | {
|
| 4776 | int order, nr;
|
| 4777 |
|
| 4778 | order = highest_order(orders);
|
| 4779 |
|
| 4780 | |
| 4781 | * To swap in a THP with nr pages, we require that its first swap_offset
|
| 4782 | * is aligned with that number, as it was when the THP was swapped out.
|
| 4783 | * This helps filter out most invalid entries.
|
| 4784 |
|
| 4785 | while (orders) {
|
| 4786 | nr = 1 << order;
|
| 4787 | if ((addr >> PAGE_SHIFT) % nr == swp_offset % nr)
|
| 4788 | break;
|
| 4789 | order = next_order(&orders, order);
|
| 4790 | }
|
| 4791 |
|
| 4792 | return orders;
|
| 4793 | }
|
| 4794 |
|
| 4795 | static unsigned long thp_swapin_suitable_orders(struct vm_fault *vmf)
|
| 4796 | {
|
| 4797 | struct vm_area_struct *vma = vmf->vma;
|
| 4798 | unsigned long orders;
|
| 4799 | unsigned long addr;
|
| 4800 | softleaf_t entry;
|
| 4801 | spinlock_t *ptl;
|
| 4802 | pte_t *pte;
|
| 4803 | int order;
|
| 4804 |
|
| 4805 | |
| 4806 | * If uffd is active for the vma we need per-page fault fidelity to
|
| 4807 | * maintain the uffd semantics.
|
| 4808 |
|
| 4809 | if (unlikely(userfaultfd_armed(vma)))
|
| 4810 | return 0;
|
| 4811 |
|
| 4812 | |
| 4813 | * A large swapped out folio could be partially or fully in zswap. We
|
| 4814 | * lack handling for such cases, so fallback to swapping in order-0
|
| 4815 | * folio.
|
| 4816 |
|
| 4817 | if (!zswap_never_enabled())
|
| 4818 | return 0;
|
| 4819 |
|
| 4820 | entry = softleaf_from_pte(vmf->orig_pte);
|
| 4821 | |
| 4822 | * Get a list of all the (large) orders below PMD_ORDER that are enabled
|
| 4823 | * and suitable for swapping THP.
|
| 4824 |
|
| 4825 | orders = thp_vma_allowable_orders(vma, vma->vm_flags, TVA_PAGEFAULT,
|
| 4826 | BIT(PMD_ORDER) - 1);
|
| 4827 | orders = thp_vma_suitable_orders(vma, vmf->address, orders);
|
| 4828 | orders = thp_swap_suitable_orders(swp_offset(entry),
|
| 4829 | vmf->address, orders);
|
| 4830 |
|
| 4831 | if (!orders)
|
| 4832 | return 0;
|
| 4833 |
|
| 4834 | pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
|
| 4835 | vmf->address & PMD_MASK, &ptl);
|
| 4836 | if (unlikely(!pte))
|
| 4837 | return 0;
|
| 4838 |
|
| 4839 | |
| 4840 | * For do_swap_page, find the highest order where the aligned range is
|
| 4841 | * completely swap entries with contiguous swap offsets.
|
| 4842 |
|
| 4843 | order = highest_order(orders);
|
| 4844 | while (orders) {
|
| 4845 | addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
|
| 4846 | if (can_swapin_thp(vmf, pte + pte_index(addr), 1 << order))
|
| 4847 | break;
|
| 4848 | order = next_order(&orders, order);
|
| 4849 | }
|
| 4850 |
|
| 4851 | pte_unmap_unlock(pte, ptl);
|
| 4852 |
|
| 4853 | return orders;
|
| 4854 | }
|
| 4855 | #else
|
| 4856 | static unsigned long thp_swapin_suitable_orders(struct vm_fault *vmf)
|
| 4857 | {
|
| 4858 | return 0;
|
| 4859 | }
|
| 4860 | #endif
|
| 4861 |
|
| 4862 |
|
| 4863 | static void check_swap_exclusive(struct folio *folio, swp_entry_t entry,
|
| 4864 | unsigned int nr_pages)
|
| 4865 | {
|
| 4866 |
|
| 4867 | do {
|
| 4868 | VM_WARN_ON_ONCE_FOLIO(__swap_count(entry) != 1, folio);
|
| 4869 | entry.val++;
|
| 4870 | } while (--nr_pages);
|
| 4871 | }
|
| 4872 |
|
| 4873 | |
| 4874 | * We enter with either the VMA lock or the mmap_lock held (see
|
| 4875 | * FAULT_FLAG_VMA_LOCK), and pte mapped but not yet locked.
|
| 4876 | * We return with pte unmapped and unlocked.
|
| 4877 | *
|
| 4878 | * When returning, the lock may have been released in the same cases
|
| 4879 | * as done by filemap_fault().
|
| 4880 |
|
| 4881 | vm_fault_t do_swap_page(struct vm_fault *vmf)
|
| 4882 | {
|
| 4883 | struct vm_area_struct *vma = vmf->vma;
|
| 4884 | struct folio *swapcache = NULL, *folio;
|
| 4885 | struct page *page;
|
| 4886 | struct swap_info_struct *si = NULL;
|
| 4887 | rmap_t rmap_flags = RMAP_NONE;
|
| 4888 | bool exclusive = false;
|
| 4889 | bool rwp_restore = false;
|
| 4890 | softleaf_t entry;
|
| 4891 | pte_t pte;
|
| 4892 | vm_fault_t ret = 0;
|
| 4893 | int nr_pages;
|
| 4894 | unsigned long page_idx;
|
| 4895 | unsigned long address;
|
| 4896 | pte_t *ptep;
|
| 4897 |
|
| 4898 | if (!pte_unmap_same(vmf))
|
| 4899 | goto out;
|
| 4900 |
|
| 4901 | entry = softleaf_from_pte(vmf->orig_pte);
|
| 4902 | if (unlikely(!softleaf_is_swap(entry))) {
|
| 4903 | if (softleaf_is_migration(entry)) {
|
| 4904 | migration_entry_wait(vma->vm_mm, vmf->pmd,
|
| 4905 | vmf->address);
|
| 4906 | } else if (softleaf_is_device_exclusive(entry)) {
|
| 4907 | vmf->page = softleaf_to_page(entry);
|
| 4908 | ret = remove_device_exclusive_entry(vmf);
|
| 4909 | } else if (softleaf_is_device_private(entry)) {
|
| 4910 | if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
|
| 4911 | |
| 4912 | * migrate_to_ram is not yet ready to operate
|
| 4913 | * under VMA lock.
|
| 4914 |
|
| 4915 | vma_end_read(vma);
|
| 4916 | ret = VM_FAULT_RETRY;
|
| 4917 | goto out;
|
| 4918 | }
|
| 4919 |
|
| 4920 | vmf->page = softleaf_to_page(entry);
|
| 4921 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
|
| 4922 | vmf->address, &vmf->ptl);
|
| 4923 | if (unlikely(!vmf->pte ||
|
| 4924 | !pte_same(ptep_get(vmf->pte),
|
| 4925 | vmf->orig_pte)))
|
| 4926 | goto unlock;
|
| 4927 |
|
| 4928 | |
| 4929 | * Get a page reference while we know the page can't be
|
| 4930 | * freed.
|
| 4931 |
|
| 4932 | if (trylock_page(vmf->page)) {
|
| 4933 | struct dev_pagemap *pgmap;
|
| 4934 |
|
| 4935 | get_page(vmf->page);
|
| 4936 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 4937 | pgmap = page_pgmap(vmf->page);
|
| 4938 | ret = pgmap->ops->migrate_to_ram(vmf);
|
| 4939 | unlock_page(vmf->page);
|
| 4940 | put_page(vmf->page);
|
| 4941 | } else {
|
| 4942 | pte_unmap(vmf->pte);
|
| 4943 | softleaf_entry_wait_on_locked(entry, vmf->ptl);
|
| 4944 | }
|
| 4945 | } else if (softleaf_is_hwpoison(entry)) {
|
| 4946 | ret = VM_FAULT_HWPOISON;
|
| 4947 | } else if (softleaf_is_marker(entry)) {
|
| 4948 | ret = handle_pte_marker(vmf);
|
| 4949 | } else {
|
| 4950 | print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
|
| 4951 | ret = VM_FAULT_SIGBUS;
|
| 4952 | }
|
| 4953 | goto out;
|
| 4954 | }
|
| 4955 |
|
| 4956 |
|
| 4957 | si = get_swap_device(entry);
|
| 4958 | if (unlikely(!si))
|
| 4959 | goto out;
|
| 4960 |
|
| 4961 | folio = swap_cache_get_folio(entry);
|
| 4962 | if (folio)
|
| 4963 | swap_update_readahead(folio, vma, vmf->address);
|
| 4964 | if (!folio) {
|
| 4965 |
|
| 4966 | if (data_race(si->flags & SWP_SYNCHRONOUS_IO))
|
| 4967 | folio = swapin_sync(entry, GFP_HIGHUSER_MOVABLE,
|
| 4968 | thp_swapin_suitable_orders(vmf) | BIT(0),
|
| 4969 | vmf, NULL, 0);
|
| 4970 | else
|
| 4971 | folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf);
|
| 4972 |
|
| 4973 | if (IS_ERR_OR_NULL(folio)) {
|
| 4974 | |
| 4975 | * Back out if somebody else faulted in this pte
|
| 4976 | * while we released the pte lock.
|
| 4977 |
|
| 4978 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
|
| 4979 | vmf->address, &vmf->ptl);
|
| 4980 | if (likely(vmf->pte &&
|
| 4981 | pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
|
| 4982 | ret = VM_FAULT_OOM;
|
| 4983 | folio = NULL;
|
| 4984 | goto unlock;
|
| 4985 | }
|
| 4986 |
|
| 4987 |
|
| 4988 | ret = VM_FAULT_MAJOR;
|
| 4989 | count_vm_event(PGMAJFAULT);
|
| 4990 | count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
|
| 4991 | }
|
| 4992 |
|
| 4993 | swapcache = folio;
|
| 4994 | ret |= folio_lock_or_retry(folio, vmf);
|
| 4995 | if (ret & VM_FAULT_RETRY)
|
| 4996 | goto out_release;
|
| 4997 |
|
| 4998 | page = folio_file_page(folio, swp_offset(entry));
|
| 4999 | |
| 5000 | * Make sure folio_free_swap() or swapoff did not release the
|
| 5001 | * swapcache from under us. The page pin, and pte_same test
|
| 5002 | * below, are not enough to exclude that. Even if it is still
|
| 5003 | * swapcache, we need to check that the page's swap has not
|
| 5004 | * changed.
|
| 5005 |
|
| 5006 | if (unlikely(!folio_matches_swap_entry(folio, entry)))
|
| 5007 | goto out_page;
|
| 5008 |
|
| 5009 | if (unlikely(PageHWPoison(page))) {
|
| 5010 | |
| 5011 | * hwpoisoned dirty swapcache pages are kept for killing
|
| 5012 | * owner processes (which may be unknown at hwpoison time)
|
| 5013 |
|
| 5014 | ret = VM_FAULT_HWPOISON;
|
| 5015 | goto out_page;
|
| 5016 | }
|
| 5017 |
|
| 5018 | |
| 5019 | * KSM sometimes has to copy on read faults, for example, if
|
| 5020 | * folio->index of non-ksm folios would be nonlinear inside the
|
| 5021 | * anon VMA -- the ksm flag is lost on actual swapout.
|
| 5022 |
|
| 5023 | folio = ksm_might_need_to_copy(folio, vma, vmf->address);
|
| 5024 | if (unlikely(!folio)) {
|
| 5025 | ret = VM_FAULT_OOM;
|
| 5026 | folio = swapcache;
|
| 5027 | goto out_page;
|
| 5028 | } else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
|
| 5029 | ret = VM_FAULT_HWPOISON;
|
| 5030 | folio = swapcache;
|
| 5031 | goto out_page;
|
| 5032 | } else if (folio != swapcache)
|
| 5033 | page = folio_page(folio, 0);
|
| 5034 |
|
| 5035 | folio_throttle_swaprate(folio, GFP_KERNEL);
|
| 5036 |
|
| 5037 | |
| 5038 | * Back out if somebody else already faulted in this pte.
|
| 5039 |
|
| 5040 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
|
| 5041 | &vmf->ptl);
|
| 5042 | if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
|
| 5043 | goto out_nomap;
|
| 5044 |
|
| 5045 | if (unlikely(!folio_test_uptodate(folio))) {
|
| 5046 | ret = VM_FAULT_SIGBUS;
|
| 5047 | goto out_nomap;
|
| 5048 | }
|
| 5049 |
|
| 5050 | nr_pages = 1;
|
| 5051 | page_idx = 0;
|
| 5052 | address = vmf->address;
|
| 5053 | ptep = vmf->pte;
|
| 5054 | if (folio_test_large(folio) && folio_test_swapcache(folio)) {
|
| 5055 | int nr = folio_nr_pages(folio);
|
| 5056 | unsigned long idx = folio_page_idx(folio, page);
|
| 5057 | unsigned long folio_start = address - idx * PAGE_SIZE;
|
| 5058 | unsigned long folio_end = folio_start + nr * PAGE_SIZE;
|
| 5059 | pte_t *folio_ptep;
|
| 5060 | pte_t folio_pte;
|
| 5061 |
|
| 5062 | if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start)))
|
| 5063 | goto check_folio;
|
| 5064 | if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end)))
|
| 5065 | goto check_folio;
|
| 5066 |
|
| 5067 | folio_ptep = vmf->pte - idx;
|
| 5068 | folio_pte = ptep_get(folio_ptep);
|
| 5069 | if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
|
| 5070 | swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
|
| 5071 | goto check_folio;
|
| 5072 |
|
| 5073 | page_idx = idx;
|
| 5074 | address = folio_start;
|
| 5075 | ptep = folio_ptep;
|
| 5076 | nr_pages = nr;
|
| 5077 | entry = folio->swap;
|
| 5078 | page = &folio->page;
|
| 5079 | }
|
| 5080 |
|
| 5081 | check_folio:
|
| 5082 | |
| 5083 | * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
|
| 5084 | * must never point at an anonymous page in the swapcache that is
|
| 5085 | * PG_anon_exclusive. Sanity check that this holds and especially, that
|
| 5086 | * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
|
| 5087 | * check after taking the PT lock and making sure that nobody
|
| 5088 | * concurrently faulted in this page and set PG_anon_exclusive.
|
| 5089 |
|
| 5090 | BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
|
| 5091 | BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
|
| 5092 |
|
| 5093 | |
| 5094 | * If a large folio already belongs to anon mapping, then we
|
| 5095 | * can just go on and map it partially.
|
| 5096 | * If not, with the large swapin check above failing, the page table
|
| 5097 | * have changed, so sub pages might got charged to the wrong cgroup,
|
| 5098 | * or even should be shmem. So we have to free it and fallback.
|
| 5099 | * Nothing should have touched it, both anon and shmem checks if a
|
| 5100 | * large folio is fully appliable before use.
|
| 5101 | *
|
| 5102 | * This will be removed once we unify folio allocation in the swap cache
|
| 5103 | * layer, where allocation of a folio stabilizes the swap entries.
|
| 5104 |
|
| 5105 | if (!folio_test_anon(folio) && folio_test_large(folio) &&
|
| 5106 | nr_pages != folio_nr_pages(folio)) {
|
| 5107 | if (!WARN_ON_ONCE(folio_test_dirty(folio)))
|
| 5108 | swap_cache_del_folio(folio);
|
| 5109 | goto out_nomap;
|
| 5110 | }
|
| 5111 |
|
| 5112 | |
| 5113 | * Check under PT lock (to protect against concurrent fork() sharing
|
| 5114 | * the swap entry concurrently) for certainly exclusive pages.
|
| 5115 |
|
| 5116 | if (!folio_test_ksm(folio)) {
|
| 5117 | |
| 5118 | * The can_swapin_thp check above ensures all PTE have
|
| 5119 | * same exclusiveness. Checking just one PTE is fine.
|
| 5120 |
|
| 5121 | exclusive = pte_swp_exclusive(vmf->orig_pte);
|
| 5122 | if (exclusive)
|
| 5123 | check_swap_exclusive(folio, entry, nr_pages);
|
| 5124 | if (folio != swapcache) {
|
| 5125 | |
| 5126 | * We have a fresh page that is not exposed to the
|
| 5127 | * swapcache -> certainly exclusive.
|
| 5128 |
|
| 5129 | exclusive = true;
|
| 5130 | } else if (exclusive && folio_test_writeback(folio) &&
|
| 5131 | data_race(si->flags & SWP_STABLE_WRITES)) {
|
| 5132 | |
| 5133 | * This is tricky: not all swap backends support
|
| 5134 | * concurrent page modifications while under writeback.
|
| 5135 | *
|
| 5136 | * So if we stumble over such a page in the swapcache
|
| 5137 | * we must not set the page exclusive, otherwise we can
|
| 5138 | * map it writable without further checks and modify it
|
| 5139 | * while still under writeback.
|
| 5140 | *
|
| 5141 | * For these problematic swap backends, simply drop the
|
| 5142 | * exclusive marker: this is perfectly fine as we start
|
| 5143 | * writeback only if we fully unmapped the page and
|
| 5144 | * there are no unexpected references on the page after
|
| 5145 | * unmapping succeeded. After fully unmapped, no
|
| 5146 | * further GUP references (FOLL_GET and FOLL_PIN) can
|
| 5147 | * appear, so dropping the exclusive marker and mapping
|
| 5148 | * it only R/O is fine.
|
| 5149 |
|
| 5150 | exclusive = false;
|
| 5151 | }
|
| 5152 | }
|
| 5153 |
|
| 5154 | |
| 5155 | * Some architectures may have to restore extra metadata to the page
|
| 5156 | * when reading from swap. This metadata may be indexed by swap entry
|
| 5157 | * so this must be called before folio_put_swap().
|
| 5158 |
|
| 5159 | arch_swap_restore(folio_swap(entry, folio), folio);
|
| 5160 |
|
| 5161 | add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
|
| 5162 | add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages);
|
| 5163 | pte = mk_pte(page, vma->vm_page_prot);
|
| 5164 | if (pte_swp_soft_dirty(vmf->orig_pte))
|
| 5165 | pte = pte_mksoft_dirty(pte);
|
| 5166 | if (pte_swp_uffd(vmf->orig_pte))
|
| 5167 | pte = pte_mkuffd(pte);
|
| 5168 |
|
| 5169 | |
| 5170 | * A page reclaimed while RWP-protected carries the uffd bit on
|
| 5171 | * its swap entry. Re-apply PAGE_NONE on swap-in so the first access
|
| 5172 | * still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD.
|
| 5173 |
|
| 5174 | if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) {
|
| 5175 | pte = pte_modify(pte, PAGE_NONE);
|
| 5176 | rwp_restore = true;
|
| 5177 | }
|
| 5178 |
|
| 5179 | |
| 5180 | * Similar logic as in do_wp_page(); however, optimize for pages that
|
| 5181 | * are certainly exclusive.
|
| 5182 | *
|
| 5183 | * Skip the write upgrade for an RWP-restored pte: it must stay
|
| 5184 | * PROT_NONE so the access retries through the RWP fault path
|
| 5185 | * (do_uffd_rwp()) rather than being made writable here.
|
| 5186 |
|
| 5187 | if (exclusive) {
|
| 5188 | if (!rwp_restore &&
|
| 5189 | (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
|
| 5190 | !pte_needs_soft_dirty_wp(vma, pte)) {
|
| 5191 | pte = pte_mkwrite(pte, vma);
|
| 5192 | if (vmf->flags & FAULT_FLAG_WRITE)
|
| 5193 | pte = pte_mkdirty(pte);
|
| 5194 | }
|
| 5195 | rmap_flags |= RMAP_EXCLUSIVE;
|
| 5196 | }
|
| 5197 | folio_ref_add(folio, nr_pages - 1);
|
| 5198 | flush_icache_pages(vma, page, nr_pages);
|
| 5199 | vmf->orig_pte = pte_advance_pfn(pte, page_idx);
|
| 5200 |
|
| 5201 |
|
| 5202 | if (unlikely(folio != swapcache)) {
|
| 5203 | folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
|
| 5204 | folio_add_lru_vma(folio, vma);
|
| 5205 | folio_put_swap(swapcache, NULL);
|
| 5206 | } else if (!folio_test_anon(folio)) {
|
| 5207 | |
| 5208 | * We currently only expect !anon folios that are fully
|
| 5209 | * mappable. See the comment after can_swapin_thp above.
|
| 5210 |
|
| 5211 | VM_WARN_ON_ONCE_FOLIO(folio_nr_pages(folio) != nr_pages, folio);
|
| 5212 | VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio);
|
| 5213 | folio_add_new_anon_rmap(folio, vma, address, rmap_flags);
|
| 5214 | folio_put_swap(folio, NULL);
|
| 5215 | } else {
|
| 5216 | VM_WARN_ON_ONCE(nr_pages != 1 && nr_pages != folio_nr_pages(folio));
|
| 5217 | folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address,
|
| 5218 | rmap_flags);
|
| 5219 | folio_put_swap(folio, nr_pages == 1 ? page : NULL);
|
| 5220 | }
|
| 5221 |
|
| 5222 | VM_BUG_ON(!folio_test_anon(folio) ||
|
| 5223 | (pte_write(pte) && !PageAnonExclusive(page)));
|
| 5224 | set_ptes(vma->vm_mm, address, ptep, pte, nr_pages);
|
| 5225 | arch_do_swap_page_nr(vma->vm_mm, vma, address,
|
| 5226 | pte, pte, nr_pages);
|
| 5227 |
|
| 5228 | |
| 5229 | * Remove the swap entry and conditionally try to free up the swapcache.
|
| 5230 | * Do it after mapping, so raced page faults will likely see the folio
|
| 5231 | * in swap cache and wait on the folio lock.
|
| 5232 |
|
| 5233 | if (should_try_to_free_swap(si, folio, vma, exclusive, vmf->flags))
|
| 5234 | folio_free_swap(folio);
|
| 5235 |
|
| 5236 | folio_unlock(folio);
|
| 5237 | if (unlikely(folio != swapcache)) {
|
| 5238 | |
| 5239 | * Hold the lock to avoid the swap entry to be reused
|
| 5240 | * until we take the PT lock for the pte_same() check
|
| 5241 | * (to avoid false positives from pte_same). For
|
| 5242 | * further safety release the lock after the folio_put_swap
|
| 5243 | * so that the swap count won't change under a
|
| 5244 | * parallel locked swapcache.
|
| 5245 |
|
| 5246 | folio_unlock(swapcache);
|
| 5247 | folio_put(swapcache);
|
| 5248 | }
|
| 5249 |
|
| 5250 | |
| 5251 | * For an RWP-restored pte, leave it PROT_NONE and let the write
|
| 5252 | * retry through the RWP fault path; do not COW it here, which would
|
| 5253 | * drop the marker for a non-exclusive page.
|
| 5254 |
|
| 5255 | if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) {
|
| 5256 | ret |= do_wp_page(vmf);
|
| 5257 | if (ret & VM_FAULT_ERROR)
|
| 5258 | ret &= VM_FAULT_ERROR;
|
| 5259 | goto out;
|
| 5260 | }
|
| 5261 |
|
| 5262 |
|
| 5263 | update_mmu_cache_range(vmf, vma, address, ptep, nr_pages);
|
| 5264 | unlock:
|
| 5265 | if (vmf->pte)
|
| 5266 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5267 | out:
|
| 5268 | if (si)
|
| 5269 | put_swap_device(si);
|
| 5270 | return ret;
|
| 5271 | out_nomap:
|
| 5272 | if (vmf->pte)
|
| 5273 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5274 | out_page:
|
| 5275 | if (folio_test_swapcache(folio))
|
| 5276 | folio_free_swap(folio);
|
| 5277 | folio_unlock(folio);
|
| 5278 | out_release:
|
| 5279 | folio_put(folio);
|
| 5280 | if (folio != swapcache) {
|
| 5281 | folio_unlock(swapcache);
|
| 5282 | folio_put(swapcache);
|
| 5283 | }
|
| 5284 | if (si)
|
| 5285 | put_swap_device(si);
|
| 5286 | return ret;
|
| 5287 | }
|
| 5288 |
|
| 5289 | static bool pte_range_none(pte_t *pte, int nr_pages)
|
| 5290 | {
|
| 5291 | int i;
|
| 5292 |
|
| 5293 | for (i = 0; i < nr_pages; i++) {
|
| 5294 | if (!pte_none(ptep_get_lockless(pte + i)))
|
| 5295 | return false;
|
| 5296 | }
|
| 5297 |
|
| 5298 | return true;
|
| 5299 | }
|
| 5300 |
|
| 5301 | static struct folio *alloc_anon_folio(struct vm_fault *vmf)
|
| 5302 | {
|
| 5303 | struct vm_area_struct *vma = vmf->vma;
|
| 5304 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 5305 | unsigned long orders;
|
| 5306 | struct folio *folio;
|
| 5307 | unsigned long addr;
|
| 5308 | pte_t *pte;
|
| 5309 | gfp_t gfp;
|
| 5310 | int order;
|
| 5311 |
|
| 5312 | |
| 5313 | * If uffd is active for the vma we need per-page fault fidelity to
|
| 5314 | * maintain the uffd semantics.
|
| 5315 |
|
| 5316 | if (unlikely(userfaultfd_armed(vma)))
|
| 5317 | goto fallback;
|
| 5318 |
|
| 5319 | |
| 5320 | * Get a list of all the (large) orders below PMD_ORDER that are enabled
|
| 5321 | * for this vma. Then filter out the orders that can't be allocated over
|
| 5322 | * the faulting address and still be fully contained in the vma.
|
| 5323 |
|
| 5324 | orders = thp_vma_allowable_orders(vma, vma->vm_flags, TVA_PAGEFAULT,
|
| 5325 | BIT(PMD_ORDER) - 1);
|
| 5326 | orders = thp_vma_suitable_orders(vma, vmf->address, orders);
|
| 5327 |
|
| 5328 | if (!orders)
|
| 5329 | goto fallback;
|
| 5330 |
|
| 5331 | pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK);
|
| 5332 | if (!pte)
|
| 5333 | return ERR_PTR(-EAGAIN);
|
| 5334 |
|
| 5335 | |
| 5336 | * Find the highest order where the aligned range is completely
|
| 5337 | * pte_none(). Note that all remaining orders will be completely
|
| 5338 | * pte_none().
|
| 5339 |
|
| 5340 | order = highest_order(orders);
|
| 5341 | while (orders) {
|
| 5342 | addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
|
| 5343 | if (pte_range_none(pte + pte_index(addr), 1 << order))
|
| 5344 | break;
|
| 5345 | order = next_order(&orders, order);
|
| 5346 | }
|
| 5347 |
|
| 5348 | pte_unmap(pte);
|
| 5349 |
|
| 5350 | if (!orders)
|
| 5351 | goto fallback;
|
| 5352 |
|
| 5353 |
|
| 5354 | gfp = vma_thp_gfp_mask(vma);
|
| 5355 | while (orders) {
|
| 5356 | addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
|
| 5357 | folio = vma_alloc_folio(gfp, order, vma, addr);
|
| 5358 | if (!folio)
|
| 5359 | goto next;
|
| 5360 | if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
|
| 5361 | count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
|
| 5362 | folio_put(folio);
|
| 5363 | goto next;
|
| 5364 | }
|
| 5365 | if (order > 1 && folio_memcg_alloc_deferred(folio)) {
|
| 5366 | folio_put(folio);
|
| 5367 | goto fallback;
|
| 5368 | }
|
| 5369 | folio_throttle_swaprate(folio, gfp);
|
| 5370 | |
| 5371 | * When a folio is not zeroed during allocation
|
| 5372 | * (__GFP_ZERO not used) or user folios require special
|
| 5373 | * handling, folio_zero_user() is used to make sure
|
| 5374 | * that the page corresponding to the faulting address
|
| 5375 | * will be hot in the cache after zeroing.
|
| 5376 |
|
| 5377 | if (user_alloc_needs_zeroing())
|
| 5378 | folio_zero_user(folio, vmf->address);
|
| 5379 | return folio;
|
| 5380 | next:
|
| 5381 | count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
|
| 5382 | order = next_order(&orders, order);
|
| 5383 | }
|
| 5384 |
|
| 5385 | fallback:
|
| 5386 | #endif
|
| 5387 | return folio_prealloc(vma->vm_mm, vma, vmf->address, true);
|
| 5388 | }
|
| 5389 |
|
| 5390 | void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte,
|
| 5391 | struct vm_area_struct *vma, unsigned long addr,
|
| 5392 | bool uffd_wp)
|
| 5393 | {
|
| 5394 | const unsigned int nr_pages = folio_nr_pages(folio);
|
| 5395 | pte_t entry = folio_mk_pte(folio, vma->vm_page_prot);
|
| 5396 |
|
| 5397 | entry = pte_sw_mkyoung(entry);
|
| 5398 |
|
| 5399 | if (vma->vm_flags & VM_WRITE)
|
| 5400 | entry = pte_mkwrite(pte_mkdirty(entry), vma);
|
| 5401 | if (uffd_wp)
|
| 5402 | entry = pte_mkuffd(entry);
|
| 5403 |
|
| 5404 | folio_ref_add(folio, nr_pages - 1);
|
| 5405 | folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
|
| 5406 | folio_add_lru_vma(folio, vma);
|
| 5407 | set_ptes(vma->vm_mm, addr, pte, entry, nr_pages);
|
| 5408 | update_mmu_cache_range(NULL, vma, addr, pte, nr_pages);
|
| 5409 | }
|
| 5410 |
|
| 5411 | static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte,
|
| 5412 | struct vm_area_struct *vma, unsigned long addr, bool uffd_wp)
|
| 5413 | {
|
| 5414 | const unsigned int order = folio_order(folio);
|
| 5415 |
|
| 5416 | map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp);
|
| 5417 | add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order);
|
| 5418 | count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC);
|
| 5419 | }
|
| 5420 |
|
| 5421 | |
| 5422 | * We enter with either the VMA lock or the mmap_lock held (see
|
| 5423 | * FAULT_FLAG_VMA_LOCK), and pte unmapped and unlocked.
|
| 5424 | * We return with the lock still held, but pte unmapped and unlocked.
|
| 5425 | * If VM_FAULT_RETRY is returned, the lock may have been released.
|
| 5426 |
|
| 5427 | static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
|
| 5428 | {
|
| 5429 | struct vm_area_struct *vma = vmf->vma;
|
| 5430 | unsigned long addr = vmf->address;
|
| 5431 | struct folio *folio;
|
| 5432 | vm_fault_t ret = 0;
|
| 5433 | int nr_pages;
|
| 5434 | pte_t entry;
|
| 5435 |
|
| 5436 |
|
| 5437 | if (vma->vm_flags & VM_SHARED)
|
| 5438 | return VM_FAULT_SIGBUS;
|
| 5439 |
|
| 5440 | |
| 5441 | * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
|
| 5442 | * be distinguished from a transient failure of pte_offset_map().
|
| 5443 |
|
| 5444 | if (pte_alloc(vma->vm_mm, vmf->pmd))
|
| 5445 | return VM_FAULT_OOM;
|
| 5446 |
|
| 5447 |
|
| 5448 | if (!(vmf->flags & FAULT_FLAG_WRITE) &&
|
| 5449 | !mm_forbids_zeropage(vma->vm_mm)) {
|
| 5450 | entry = pte_mkspecial(pfn_pte(zero_pfn(vmf->address),
|
| 5451 | vma->vm_page_prot));
|
| 5452 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
|
| 5453 | vmf->address, &vmf->ptl);
|
| 5454 | if (!vmf->pte)
|
| 5455 | goto unlock;
|
| 5456 | if (vmf_pte_changed(vmf)) {
|
| 5457 | update_mmu_tlb(vma, vmf->address, vmf->pte);
|
| 5458 | goto unlock;
|
| 5459 | }
|
| 5460 | ret = check_stable_address_space(vma->vm_mm);
|
| 5461 | if (ret)
|
| 5462 | goto unlock;
|
| 5463 |
|
| 5464 | if (userfaultfd_missing(vma)) {
|
| 5465 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5466 | return handle_userfault(vmf, VM_UFFD_MISSING);
|
| 5467 | }
|
| 5468 | if (vmf_orig_pte_uffd_wp(vmf))
|
| 5469 | entry = pte_mkuffd(entry);
|
| 5470 | set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
|
| 5471 |
|
| 5472 |
|
| 5473 | update_mmu_cache(vma, addr, vmf->pte);
|
| 5474 | goto unlock;
|
| 5475 | }
|
| 5476 |
|
| 5477 |
|
| 5478 | ret = vmf_anon_prepare(vmf);
|
| 5479 | if (ret)
|
| 5480 | return ret;
|
| 5481 |
|
| 5482 | folio = alloc_anon_folio(vmf);
|
| 5483 | if (IS_ERR(folio))
|
| 5484 | return 0;
|
| 5485 | if (!folio)
|
| 5486 | goto oom;
|
| 5487 |
|
| 5488 | nr_pages = folio_nr_pages(folio);
|
| 5489 | addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
|
| 5490 |
|
| 5491 | |
| 5492 | * The memory barrier inside __folio_mark_uptodate makes sure that
|
| 5493 | * preceding stores to the page contents become visible before
|
| 5494 | * the set_pte_at() write.
|
| 5495 |
|
| 5496 | __folio_mark_uptodate(folio);
|
| 5497 |
|
| 5498 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
|
| 5499 | if (!vmf->pte)
|
| 5500 | goto release;
|
| 5501 | if (nr_pages == 1 && vmf_pte_changed(vmf)) {
|
| 5502 | update_mmu_tlb(vma, addr, vmf->pte);
|
| 5503 | goto release;
|
| 5504 | } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
|
| 5505 | update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
|
| 5506 | goto release;
|
| 5507 | }
|
| 5508 |
|
| 5509 | ret = check_stable_address_space(vma->vm_mm);
|
| 5510 | if (ret)
|
| 5511 | goto release;
|
| 5512 |
|
| 5513 |
|
| 5514 | if (userfaultfd_missing(vma)) {
|
| 5515 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5516 | folio_put(folio);
|
| 5517 | return handle_userfault(vmf, VM_UFFD_MISSING);
|
| 5518 | }
|
| 5519 | map_anon_folio_pte_pf(folio, vmf->pte, vma, addr,
|
| 5520 | vmf_orig_pte_uffd_wp(vmf));
|
| 5521 | unlock:
|
| 5522 | if (vmf->pte)
|
| 5523 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5524 | return ret;
|
| 5525 | release:
|
| 5526 | folio_put(folio);
|
| 5527 | goto unlock;
|
| 5528 | oom:
|
| 5529 | return VM_FAULT_OOM;
|
| 5530 | }
|
| 5531 |
|
| 5532 | |
| 5533 | * Either the VMA lock or the mmap_lock must have been held on entry
|
| 5534 | * (see FAULT_FLAG_VMA_LOCK) and may have been released depending on
|
| 5535 | * flags and vma->vm_ops->fault() return value.
|
| 5536 | * See filemap_fault() and __folio_lock_or_retry().
|
| 5537 |
|
| 5538 | static vm_fault_t __do_fault(struct vm_fault *vmf)
|
| 5539 | {
|
| 5540 | struct vm_area_struct *vma = vmf->vma;
|
| 5541 | struct folio *folio;
|
| 5542 | vm_fault_t ret;
|
| 5543 |
|
| 5544 | |
| 5545 | * Preallocate pte before we take folio lock because this might lead to
|
| 5546 | * deadlocks for memcg reclaim which waits for folios under writeback:
|
| 5547 | * folio_lock(A)
|
| 5548 | * folio_set_writeback(A)
|
| 5549 | * folio_unlock(A)
|
| 5550 | * folio_lock(B)
|
| 5551 | * folio_lock(B)
|
| 5552 | * pte_alloc_one
|
| 5553 | * shrink_folio_list
|
| 5554 | * folio_wait_writeback(A)
|
| 5555 | * folio_set_writeback(B)
|
| 5556 | * folio_unlock(B)
|
| 5557 | * # flush A, B to clear the writeback
|
| 5558 |
|
| 5559 | if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
|
| 5560 | vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
|
| 5561 | if (!vmf->prealloc_pte)
|
| 5562 | return VM_FAULT_OOM;
|
| 5563 | }
|
| 5564 |
|
| 5565 | ret = vma->vm_ops->fault(vmf);
|
| 5566 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
|
| 5567 | VM_FAULT_DONE_COW)))
|
| 5568 | return ret;
|
| 5569 |
|
| 5570 | folio = page_folio(vmf->page);
|
| 5571 | if (unlikely(PageHWPoison(vmf->page))) {
|
| 5572 | vm_fault_t poisonret = VM_FAULT_HWPOISON;
|
| 5573 | if (ret & VM_FAULT_LOCKED) {
|
| 5574 | if (folio_mapped(folio))
|
| 5575 | unmap_mapping_folio(folio);
|
| 5576 |
|
| 5577 | if (mapping_evict_folio(folio->mapping, folio))
|
| 5578 | poisonret = VM_FAULT_NOPAGE;
|
| 5579 | folio_unlock(folio);
|
| 5580 | }
|
| 5581 | folio_put(folio);
|
| 5582 | vmf->page = NULL;
|
| 5583 | return poisonret;
|
| 5584 | }
|
| 5585 |
|
| 5586 | if (unlikely(!(ret & VM_FAULT_LOCKED)))
|
| 5587 | folio_lock(folio);
|
| 5588 | else
|
| 5589 | VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);
|
| 5590 |
|
| 5591 | return ret;
|
| 5592 | }
|
| 5593 |
|
| 5594 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 5595 | static void deposit_prealloc_pte(struct vm_fault *vmf)
|
| 5596 | {
|
| 5597 | struct vm_area_struct *vma = vmf->vma;
|
| 5598 |
|
| 5599 | pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
|
| 5600 | |
| 5601 | * We are going to consume the prealloc table,
|
| 5602 | * count that as nr_ptes.
|
| 5603 |
|
| 5604 | mm_inc_nr_ptes(vma->vm_mm);
|
| 5605 | vmf->prealloc_pte = NULL;
|
| 5606 | }
|
| 5607 |
|
| 5608 | vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page)
|
| 5609 | {
|
| 5610 | struct vm_area_struct *vma = vmf->vma;
|
| 5611 | bool write = vmf->flags & FAULT_FLAG_WRITE;
|
| 5612 | unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
|
| 5613 | pmd_t entry;
|
| 5614 | vm_fault_t ret = VM_FAULT_FALLBACK;
|
| 5615 |
|
| 5616 | |
| 5617 | * It is too late to allocate a small folio, we already have a large
|
| 5618 | * folio in the pagecache: especially s390 KVM cannot tolerate any
|
| 5619 | * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any
|
| 5620 | * PMD mappings if THPs are disabled. As we already have a THP,
|
| 5621 | * behave as if we are forcing a collapse.
|
| 5622 |
|
| 5623 | if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags,
|
| 5624 | true))
|
| 5625 | return ret;
|
| 5626 |
|
| 5627 | if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
|
| 5628 | return ret;
|
| 5629 |
|
| 5630 | if (!is_pmd_order(folio_order(folio)))
|
| 5631 | return ret;
|
| 5632 | page = &folio->page;
|
| 5633 |
|
| 5634 | |
| 5635 | * Just backoff if any subpage of a THP is corrupted otherwise
|
| 5636 | * the corrupted page may mapped by PMD silently to escape the
|
| 5637 | * check. This kind of THP just can be PTE mapped. Access to
|
| 5638 | * the corrupted subpage should trigger SIGBUS as expected.
|
| 5639 |
|
| 5640 | if (unlikely(folio_test_has_hwpoisoned(folio)))
|
| 5641 | return ret;
|
| 5642 |
|
| 5643 | |
| 5644 | * Archs like ppc64 need additional space to store information
|
| 5645 | * related to pte entry. Use the preallocated table for that.
|
| 5646 |
|
| 5647 | if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
|
| 5648 | vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
|
| 5649 | if (!vmf->prealloc_pte)
|
| 5650 | return VM_FAULT_OOM;
|
| 5651 | }
|
| 5652 |
|
| 5653 | vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
|
| 5654 | if (unlikely(!pmd_none(*vmf->pmd)))
|
| 5655 | goto out;
|
| 5656 |
|
| 5657 | flush_icache_pages(vma, page, HPAGE_PMD_NR);
|
| 5658 |
|
| 5659 | entry = folio_mk_pmd(folio, vma->vm_page_prot);
|
| 5660 | if (write)
|
| 5661 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
|
| 5662 |
|
| 5663 | add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR);
|
| 5664 | folio_add_file_rmap_pmd(folio, page, vma);
|
| 5665 |
|
| 5666 | |
| 5667 | * deposit and withdraw with pmd lock held
|
| 5668 |
|
| 5669 | if (arch_needs_pgtable_deposit())
|
| 5670 | deposit_prealloc_pte(vmf);
|
| 5671 |
|
| 5672 | set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
|
| 5673 |
|
| 5674 | update_mmu_cache_pmd(vma, haddr, vmf->pmd);
|
| 5675 |
|
| 5676 |
|
| 5677 | ret = 0;
|
| 5678 | count_vm_event(THP_FILE_MAPPED);
|
| 5679 | out:
|
| 5680 | spin_unlock(vmf->ptl);
|
| 5681 | return ret;
|
| 5682 | }
|
| 5683 | #else
|
| 5684 | vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page)
|
| 5685 | {
|
| 5686 | return VM_FAULT_FALLBACK;
|
| 5687 | }
|
| 5688 | #endif
|
| 5689 |
|
| 5690 | |
| 5691 | * set_pte_range - Set a range of PTEs to point to pages in a folio.
|
| 5692 | * @vmf: Fault description.
|
| 5693 | * @folio: The folio that contains @page.
|
| 5694 | * @page: The first page to create a PTE for.
|
| 5695 | * @nr: The number of PTEs to create.
|
| 5696 | * @addr: The first address to create a PTE for.
|
| 5697 |
|
| 5698 | void set_pte_range(struct vm_fault *vmf, struct folio *folio,
|
| 5699 | struct page *page, unsigned int nr, unsigned long addr)
|
| 5700 | {
|
| 5701 | struct vm_area_struct *vma = vmf->vma;
|
| 5702 | bool write = vmf->flags & FAULT_FLAG_WRITE;
|
| 5703 | bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE);
|
| 5704 | pte_t entry;
|
| 5705 |
|
| 5706 | flush_icache_pages(vma, page, nr);
|
| 5707 | entry = mk_pte(page, vma->vm_page_prot);
|
| 5708 |
|
| 5709 | if (prefault && arch_wants_old_prefaulted_pte())
|
| 5710 | entry = pte_mkold(entry);
|
| 5711 | else
|
| 5712 | entry = pte_sw_mkyoung(entry);
|
| 5713 |
|
| 5714 | if (write)
|
| 5715 | entry = maybe_mkwrite(pte_mkdirty(entry), vma);
|
| 5716 | else if (pte_write(entry) && folio_test_dirty(folio))
|
| 5717 | entry = pte_mkdirty(entry);
|
| 5718 | if (unlikely(vmf_orig_pte_uffd_wp(vmf)))
|
| 5719 | entry = pte_mkuffd(entry);
|
| 5720 |
|
| 5721 | if (write && !(vma->vm_flags & VM_SHARED)) {
|
| 5722 | VM_BUG_ON_FOLIO(nr != 1, folio);
|
| 5723 | folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
|
| 5724 | folio_add_lru_vma(folio, vma);
|
| 5725 | } else {
|
| 5726 | folio_add_file_rmap_ptes(folio, page, nr, vma);
|
| 5727 | }
|
| 5728 | set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
|
| 5729 |
|
| 5730 |
|
| 5731 | update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
|
| 5732 | }
|
| 5733 |
|
| 5734 | static bool vmf_pte_changed(struct vm_fault *vmf)
|
| 5735 | {
|
| 5736 | if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
|
| 5737 | return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
|
| 5738 |
|
| 5739 | return !pte_none(ptep_get(vmf->pte));
|
| 5740 | }
|
| 5741 |
|
| 5742 | |
| 5743 | * finish_fault - finish page fault once we have prepared the page to fault
|
| 5744 | *
|
| 5745 | * @vmf: structure describing the fault
|
| 5746 | *
|
| 5747 | * This function handles all that is needed to finish a page fault once the
|
| 5748 | * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
|
| 5749 | * given page, adds reverse page mapping, handles memcg charges and LRU
|
| 5750 | * addition.
|
| 5751 | *
|
| 5752 | * The function expects the page to be locked and on success it consumes a
|
| 5753 | * reference of a page being mapped (for the PTE which maps it).
|
| 5754 | *
|
| 5755 | * Return: %0 on success, %VM_FAULT_ code in case of error.
|
| 5756 |
|
| 5757 | vm_fault_t finish_fault(struct vm_fault *vmf)
|
| 5758 | {
|
| 5759 | struct vm_area_struct *vma = vmf->vma;
|
| 5760 | struct page *page;
|
| 5761 | struct folio *folio;
|
| 5762 | vm_fault_t ret;
|
| 5763 | bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) &&
|
| 5764 | !(vma->vm_flags & VM_SHARED);
|
| 5765 | int type, nr_pages;
|
| 5766 | unsigned long addr;
|
| 5767 | bool needs_fallback = false;
|
| 5768 |
|
| 5769 | fallback:
|
| 5770 | addr = vmf->address;
|
| 5771 |
|
| 5772 |
|
| 5773 | if (is_cow)
|
| 5774 | page = vmf->cow_page;
|
| 5775 | else
|
| 5776 | page = vmf->page;
|
| 5777 |
|
| 5778 | folio = page_folio(page);
|
| 5779 | |
| 5780 | * check even for read faults because we might have lost our CoWed
|
| 5781 | * page
|
| 5782 |
|
| 5783 | if (!(vma->vm_flags & VM_SHARED)) {
|
| 5784 | ret = check_stable_address_space(vma->vm_mm);
|
| 5785 | if (ret)
|
| 5786 | return ret;
|
| 5787 | }
|
| 5788 |
|
| 5789 | if (!needs_fallback && vma->vm_file) {
|
| 5790 | struct address_space *mapping = vma->vm_file->f_mapping;
|
| 5791 | pgoff_t file_end;
|
| 5792 |
|
| 5793 | file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
|
| 5794 |
|
| 5795 | |
| 5796 | * Do not allow to map with PTEs beyond i_size and with PMD
|
| 5797 | * across i_size to preserve SIGBUS semantics.
|
| 5798 | *
|
| 5799 | * Make an exception for shmem/tmpfs that for long time
|
| 5800 | * intentionally mapped with PMDs across i_size.
|
| 5801 |
|
| 5802 | needs_fallback = !shmem_mapping(mapping) &&
|
| 5803 | file_end < folio_next_index(folio);
|
| 5804 | }
|
| 5805 |
|
| 5806 | if (pmd_none(*vmf->pmd)) {
|
| 5807 | if (!needs_fallback && folio_test_pmd_mappable(folio)) {
|
| 5808 | ret = do_set_pmd(vmf, folio, page);
|
| 5809 | if (ret != VM_FAULT_FALLBACK)
|
| 5810 | return ret;
|
| 5811 | }
|
| 5812 |
|
| 5813 | if (vmf->prealloc_pte)
|
| 5814 | pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
|
| 5815 | else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
|
| 5816 | return VM_FAULT_OOM;
|
| 5817 | }
|
| 5818 |
|
| 5819 | nr_pages = folio_nr_pages(folio);
|
| 5820 |
|
| 5821 |
|
| 5822 | if (unlikely(userfaultfd_armed(vma)) || unlikely(needs_fallback)) {
|
| 5823 | nr_pages = 1;
|
| 5824 | } else if (nr_pages > 1) {
|
| 5825 | pgoff_t idx = folio_page_idx(folio, page);
|
| 5826 |
|
| 5827 | pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma);
|
| 5828 |
|
| 5829 | pgoff_t pte_off = pte_index(vmf->address);
|
| 5830 |
|
| 5831 | |
| 5832 | * Fallback to per-page fault in case the folio size in page
|
| 5833 | * cache beyond the VMA limits and PMD pagetable limits.
|
| 5834 |
|
| 5835 | if (unlikely(vma_off < idx ||
|
| 5836 | vma_off + (nr_pages - idx) > vma_pages(vma) ||
|
| 5837 | pte_off < idx ||
|
| 5838 | pte_off + (nr_pages - idx) > PTRS_PER_PTE)) {
|
| 5839 | nr_pages = 1;
|
| 5840 | } else {
|
| 5841 |
|
| 5842 | addr = vmf->address - idx * PAGE_SIZE;
|
| 5843 | page = &folio->page;
|
| 5844 | }
|
| 5845 | }
|
| 5846 |
|
| 5847 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
|
| 5848 | addr, &vmf->ptl);
|
| 5849 | if (!vmf->pte)
|
| 5850 | return VM_FAULT_NOPAGE;
|
| 5851 |
|
| 5852 |
|
| 5853 | if (nr_pages == 1 && unlikely(vmf_pte_changed(vmf))) {
|
| 5854 | update_mmu_tlb(vma, addr, vmf->pte);
|
| 5855 | ret = VM_FAULT_NOPAGE;
|
| 5856 | goto unlock;
|
| 5857 | } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
|
| 5858 | needs_fallback = true;
|
| 5859 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5860 | goto fallback;
|
| 5861 | }
|
| 5862 |
|
| 5863 | folio_ref_add(folio, nr_pages - 1);
|
| 5864 | set_pte_range(vmf, folio, page, nr_pages, addr);
|
| 5865 | type = is_cow ? MM_ANONPAGES : mm_counter_file(folio);
|
| 5866 | add_mm_counter(vma->vm_mm, type, nr_pages);
|
| 5867 | ret = 0;
|
| 5868 |
|
| 5869 | unlock:
|
| 5870 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 5871 | return ret;
|
| 5872 | }
|
| 5873 |
|
| 5874 | static unsigned long fault_around_pages __read_mostly =
|
| 5875 | 65536 >> PAGE_SHIFT;
|
| 5876 |
|
| 5877 | #ifdef CONFIG_DEBUG_FS
|
| 5878 | static int fault_around_bytes_get(void *data, u64 *val)
|
| 5879 | {
|
| 5880 | *val = fault_around_pages << PAGE_SHIFT;
|
| 5881 | return 0;
|
| 5882 | }
|
| 5883 |
|
| 5884 | |
| 5885 | * fault_around_bytes must be rounded down to the nearest page order as it's
|
| 5886 | * what do_fault_around() expects to see.
|
| 5887 |
|
| 5888 | static int fault_around_bytes_set(void *data, u64 val)
|
| 5889 | {
|
| 5890 | if (val / PAGE_SIZE > PTRS_PER_PTE)
|
| 5891 | return -EINVAL;
|
| 5892 |
|
| 5893 | |
| 5894 | * The minimum value is 1 page, however this results in no fault-around
|
| 5895 | * at all. See should_fault_around().
|
| 5896 |
|
| 5897 | val = max(val, PAGE_SIZE);
|
| 5898 | fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT;
|
| 5899 |
|
| 5900 | return 0;
|
| 5901 | }
|
| 5902 | DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
|
| 5903 | fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
|
| 5904 |
|
| 5905 | static int __init fault_around_debugfs(void)
|
| 5906 | {
|
| 5907 | debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
|
| 5908 | &fault_around_bytes_fops);
|
| 5909 | return 0;
|
| 5910 | }
|
| 5911 | late_initcall(fault_around_debugfs);
|
| 5912 | #endif
|
| 5913 |
|
| 5914 | |
| 5915 | * do_fault_around() tries to map few pages around the fault address. The hope
|
| 5916 | * is that the pages will be needed soon and this will lower the number of
|
| 5917 | * faults to handle.
|
| 5918 | *
|
| 5919 | * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
|
| 5920 | * not ready to be mapped: not up-to-date, locked, etc.
|
| 5921 | *
|
| 5922 | * This function doesn't cross VMA or page table boundaries, in order to call
|
| 5923 | * map_pages() and acquire a PTE lock only once.
|
| 5924 | *
|
| 5925 | * fault_around_pages defines how many pages we'll try to map.
|
| 5926 | * do_fault_around() expects it to be set to a power of two less than or equal
|
| 5927 | * to PTRS_PER_PTE.
|
| 5928 | *
|
| 5929 | * The virtual address of the area that we map is naturally aligned to
|
| 5930 | * fault_around_pages * PAGE_SIZE rounded down to the machine page size
|
| 5931 | * (and therefore to page order). This way it's easier to guarantee
|
| 5932 | * that we don't cross page table boundaries.
|
| 5933 |
|
| 5934 | static vm_fault_t do_fault_around(struct vm_fault *vmf)
|
| 5935 | {
|
| 5936 | pgoff_t nr_pages = READ_ONCE(fault_around_pages);
|
| 5937 | pgoff_t pte_off = pte_index(vmf->address);
|
| 5938 |
|
| 5939 | pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma);
|
| 5940 | pgoff_t from_pte, to_pte;
|
| 5941 | vm_fault_t ret;
|
| 5942 |
|
| 5943 |
|
| 5944 | from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
|
| 5945 | pte_off - min(pte_off, vma_off));
|
| 5946 |
|
| 5947 |
|
| 5948 | to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
|
| 5949 | pte_off + vma_pages(vmf->vma) - vma_off) - 1;
|
| 5950 |
|
| 5951 | if (pmd_none(*vmf->pmd)) {
|
| 5952 | vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
|
| 5953 | if (!vmf->prealloc_pte)
|
| 5954 | return VM_FAULT_OOM;
|
| 5955 | }
|
| 5956 |
|
| 5957 | rcu_read_lock();
|
| 5958 | ret = vmf->vma->vm_ops->map_pages(vmf,
|
| 5959 | vmf->pgoff + from_pte - pte_off,
|
| 5960 | vmf->pgoff + to_pte - pte_off);
|
| 5961 | rcu_read_unlock();
|
| 5962 |
|
| 5963 | return ret;
|
| 5964 | }
|
| 5965 |
|
| 5966 |
|
| 5967 | static inline bool should_fault_around(struct vm_fault *vmf)
|
| 5968 | {
|
| 5969 |
|
| 5970 | if (!vmf->vma->vm_ops->map_pages)
|
| 5971 | return false;
|
| 5972 |
|
| 5973 | if (uffd_disable_fault_around(vmf->vma))
|
| 5974 | return false;
|
| 5975 |
|
| 5976 |
|
| 5977 | return fault_around_pages > 1;
|
| 5978 | }
|
| 5979 |
|
| 5980 | static vm_fault_t do_read_fault(struct vm_fault *vmf)
|
| 5981 | {
|
| 5982 | vm_fault_t ret = 0;
|
| 5983 | struct folio *folio;
|
| 5984 |
|
| 5985 | |
| 5986 | * Let's call ->map_pages() first and use ->fault() as fallback
|
| 5987 | * if page by the offset is not ready to be mapped (cold cache or
|
| 5988 | * something).
|
| 5989 |
|
| 5990 | if (should_fault_around(vmf)) {
|
| 5991 | ret = do_fault_around(vmf);
|
| 5992 | if (ret)
|
| 5993 | return ret;
|
| 5994 | }
|
| 5995 |
|
| 5996 | ret = vmf_can_call_fault(vmf);
|
| 5997 | if (ret)
|
| 5998 | return ret;
|
| 5999 |
|
| 6000 | ret = __do_fault(vmf);
|
| 6001 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
|
| 6002 | return ret;
|
| 6003 |
|
| 6004 | ret |= finish_fault(vmf);
|
| 6005 | folio = page_folio(vmf->page);
|
| 6006 | folio_unlock(folio);
|
| 6007 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
|
| 6008 | folio_put(folio);
|
| 6009 | return ret;
|
| 6010 | }
|
| 6011 |
|
| 6012 | static vm_fault_t do_cow_fault(struct vm_fault *vmf)
|
| 6013 | {
|
| 6014 | struct vm_area_struct *vma = vmf->vma;
|
| 6015 | struct folio *folio;
|
| 6016 | vm_fault_t ret;
|
| 6017 |
|
| 6018 | ret = vmf_can_call_fault(vmf);
|
| 6019 | if (!ret)
|
| 6020 | ret = vmf_anon_prepare(vmf);
|
| 6021 | if (ret)
|
| 6022 | return ret;
|
| 6023 |
|
| 6024 | folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false);
|
| 6025 | if (!folio)
|
| 6026 | return VM_FAULT_OOM;
|
| 6027 |
|
| 6028 | vmf->cow_page = &folio->page;
|
| 6029 |
|
| 6030 | ret = __do_fault(vmf);
|
| 6031 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
|
| 6032 | goto uncharge_out;
|
| 6033 | if (ret & VM_FAULT_DONE_COW)
|
| 6034 | return ret;
|
| 6035 |
|
| 6036 | if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) {
|
| 6037 | ret = VM_FAULT_HWPOISON;
|
| 6038 | goto unlock;
|
| 6039 | }
|
| 6040 | __folio_mark_uptodate(folio);
|
| 6041 |
|
| 6042 | ret |= finish_fault(vmf);
|
| 6043 | unlock:
|
| 6044 | unlock_page(vmf->page);
|
| 6045 | put_page(vmf->page);
|
| 6046 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
|
| 6047 | goto uncharge_out;
|
| 6048 | return ret;
|
| 6049 | uncharge_out:
|
| 6050 | folio_put(folio);
|
| 6051 | return ret;
|
| 6052 | }
|
| 6053 |
|
| 6054 | static vm_fault_t do_shared_fault(struct vm_fault *vmf)
|
| 6055 | {
|
| 6056 | struct vm_area_struct *vma = vmf->vma;
|
| 6057 | vm_fault_t ret, tmp;
|
| 6058 | struct folio *folio;
|
| 6059 |
|
| 6060 | ret = vmf_can_call_fault(vmf);
|
| 6061 | if (ret)
|
| 6062 | return ret;
|
| 6063 |
|
| 6064 | ret = __do_fault(vmf);
|
| 6065 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
|
| 6066 | return ret;
|
| 6067 |
|
| 6068 | folio = page_folio(vmf->page);
|
| 6069 |
|
| 6070 | |
| 6071 | * Check if the backing address space wants to know that the page is
|
| 6072 | * about to become writable
|
| 6073 |
|
| 6074 | if (vma->vm_ops->page_mkwrite) {
|
| 6075 | folio_unlock(folio);
|
| 6076 | tmp = do_page_mkwrite(vmf, folio);
|
| 6077 | if (unlikely(!tmp ||
|
| 6078 | (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
|
| 6079 | folio_put(folio);
|
| 6080 | return tmp;
|
| 6081 | }
|
| 6082 | }
|
| 6083 |
|
| 6084 | ret |= finish_fault(vmf);
|
| 6085 | if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
|
| 6086 | VM_FAULT_RETRY))) {
|
| 6087 | folio_unlock(folio);
|
| 6088 | folio_put(folio);
|
| 6089 | return ret;
|
| 6090 | }
|
| 6091 |
|
| 6092 | ret |= fault_dirty_shared_page(vmf);
|
| 6093 | return ret;
|
| 6094 | }
|
| 6095 |
|
| 6096 | |
| 6097 | * We enter with either the VMA lock or the mmap_lock held (see
|
| 6098 | * FAULT_FLAG_VMA_LOCK).
|
| 6099 | * The lock may have been released depending on flags and our
|
| 6100 | * return value. See filemap_fault() and __folio_lock_or_retry().
|
| 6101 | * If the lock is released, vma may become invalid (for example
|
| 6102 | * by other thread calling munmap()).
|
| 6103 |
|
| 6104 | static vm_fault_t do_fault(struct vm_fault *vmf)
|
| 6105 | {
|
| 6106 | struct vm_area_struct *vma = vmf->vma;
|
| 6107 | struct mm_struct *vm_mm = vma->vm_mm;
|
| 6108 | vm_fault_t ret;
|
| 6109 |
|
| 6110 | |
| 6111 | * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
|
| 6112 |
|
| 6113 | if (!vma->vm_ops->fault) {
|
| 6114 | vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
|
| 6115 | vmf->address, &vmf->ptl);
|
| 6116 | if (unlikely(!vmf->pte))
|
| 6117 | ret = VM_FAULT_SIGBUS;
|
| 6118 | else {
|
| 6119 | |
| 6120 | * Make sure this is not a temporary clearing of pte
|
| 6121 | * by holding ptl and checking again. A R/M/W update
|
| 6122 | * of pte involves: take ptl, clearing the pte so that
|
| 6123 | * we don't have concurrent modification by hardware
|
| 6124 | * followed by an update.
|
| 6125 |
|
| 6126 | if (unlikely(pte_none(ptep_get(vmf->pte))))
|
| 6127 | ret = VM_FAULT_SIGBUS;
|
| 6128 | else
|
| 6129 | ret = VM_FAULT_NOPAGE;
|
| 6130 |
|
| 6131 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6132 | }
|
| 6133 | } else if (!(vmf->flags & FAULT_FLAG_WRITE))
|
| 6134 | ret = do_read_fault(vmf);
|
| 6135 | else if (!(vma->vm_flags & VM_SHARED))
|
| 6136 | ret = do_cow_fault(vmf);
|
| 6137 | else
|
| 6138 | ret = do_shared_fault(vmf);
|
| 6139 |
|
| 6140 |
|
| 6141 | if (vmf->prealloc_pte) {
|
| 6142 | pte_free(vm_mm, vmf->prealloc_pte);
|
| 6143 | vmf->prealloc_pte = NULL;
|
| 6144 | }
|
| 6145 | return ret;
|
| 6146 | }
|
| 6147 |
|
| 6148 | int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
|
| 6149 | unsigned long addr, int *flags,
|
| 6150 | bool writable, int *last_cpupid)
|
| 6151 | {
|
| 6152 | struct vm_area_struct *vma = vmf->vma;
|
| 6153 |
|
| 6154 | |
| 6155 | * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
|
| 6156 | * much anyway since they can be in shared cache state. This misses
|
| 6157 | * the case where a mapping is writable but the process never writes
|
| 6158 | * to it but pte_write gets cleared during protection updates and
|
| 6159 | * pte_dirty has unpredictable behaviour between PTE scan updates,
|
| 6160 | * background writeback, dirty balancing and application behaviour.
|
| 6161 |
|
| 6162 | if (!writable)
|
| 6163 | *flags |= TNF_NO_GROUP;
|
| 6164 |
|
| 6165 | |
| 6166 | * Flag if the folio is shared between multiple address spaces. This
|
| 6167 | * is later used when determining whether to group tasks together
|
| 6168 |
|
| 6169 | if (folio_maybe_mapped_shared(folio) && (vma->vm_flags & VM_SHARED))
|
| 6170 | *flags |= TNF_SHARED;
|
| 6171 | |
| 6172 | * For memory tiering mode, cpupid of slow memory page is used
|
| 6173 | * to record page access time. So use default value.
|
| 6174 |
|
| 6175 | if (folio_use_access_time(folio))
|
| 6176 | *last_cpupid = (-1 & LAST_CPUPID_MASK);
|
| 6177 | else
|
| 6178 | *last_cpupid = folio_last_cpupid(folio);
|
| 6179 |
|
| 6180 |
|
| 6181 | vma_set_access_pid_bit(vma);
|
| 6182 |
|
| 6183 | count_vm_numa_event(NUMA_HINT_FAULTS);
|
| 6184 | #ifdef CONFIG_NUMA_BALANCING
|
| 6185 | count_memcg_folio_events(folio, NUMA_HINT_FAULTS, 1);
|
| 6186 | #endif
|
| 6187 | if (folio_nid(folio) == numa_node_id()) {
|
| 6188 | count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
|
| 6189 | *flags |= TNF_FAULT_LOCAL;
|
| 6190 | }
|
| 6191 |
|
| 6192 | return mpol_misplaced(folio, vmf, addr);
|
| 6193 | }
|
| 6194 |
|
| 6195 | static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
|
| 6196 | unsigned long fault_addr, pte_t *fault_pte,
|
| 6197 | bool writable)
|
| 6198 | {
|
| 6199 | pte_t pte, old_pte;
|
| 6200 |
|
| 6201 | old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte);
|
| 6202 | pte = pte_modify(old_pte, vma->vm_page_prot);
|
| 6203 | pte = pte_mkyoung(pte);
|
| 6204 | if (writable)
|
| 6205 | pte = pte_mkwrite(pte, vma);
|
| 6206 | ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte);
|
| 6207 | update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1);
|
| 6208 | }
|
| 6209 |
|
| 6210 | static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
|
| 6211 | struct folio *folio, pte_t fault_pte,
|
| 6212 | bool ignore_writable, bool pte_write_upgrade)
|
| 6213 | {
|
| 6214 | int nr = pte_pfn(fault_pte) - folio_pfn(folio);
|
| 6215 | unsigned long start, end, addr = vmf->address;
|
| 6216 | unsigned long addr_start = addr - (nr << PAGE_SHIFT);
|
| 6217 | unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE);
|
| 6218 | pte_t *start_ptep;
|
| 6219 |
|
| 6220 |
|
| 6221 | start = max3(addr_start, pt_start, vma->vm_start);
|
| 6222 | end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE,
|
| 6223 | vma->vm_end);
|
| 6224 | start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT);
|
| 6225 |
|
| 6226 |
|
| 6227 | for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) {
|
| 6228 | pte_t ptent = ptep_get(start_ptep);
|
| 6229 | bool writable = false;
|
| 6230 |
|
| 6231 | if (!pte_present(ptent) || !pte_protnone(ptent))
|
| 6232 | continue;
|
| 6233 |
|
| 6234 | |
| 6235 | * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a
|
| 6236 | * marker. Leave them alone -- rewriting to vm_page_prot would
|
| 6237 | * stop the RWP trap. Gate on userfaultfd_rwp(vma) too:
|
| 6238 | * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs
|
| 6239 | * when applying PROT_NONE, and those still need rebuilding.
|
| 6240 |
|
| 6241 | if (userfaultfd_rwp(vma) && pte_uffd(ptent))
|
| 6242 | continue;
|
| 6243 |
|
| 6244 | if (pfn_folio(pte_pfn(ptent)) != folio)
|
| 6245 | continue;
|
| 6246 |
|
| 6247 | if (!ignore_writable) {
|
| 6248 | ptent = pte_modify(ptent, vma->vm_page_prot);
|
| 6249 | writable = pte_write(ptent);
|
| 6250 | if (!writable && pte_write_upgrade &&
|
| 6251 | can_change_pte_writable(vma, addr, ptent))
|
| 6252 | writable = true;
|
| 6253 | }
|
| 6254 |
|
| 6255 | numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable);
|
| 6256 | }
|
| 6257 | }
|
| 6258 |
|
| 6259 | static vm_fault_t do_uffd_rwp(struct vm_fault *vmf)
|
| 6260 | {
|
| 6261 | pte_t pte;
|
| 6262 |
|
| 6263 | if (!userfaultfd_rwp_async(vmf->vma)) {
|
| 6264 |
|
| 6265 | pte_unmap(vmf->pte);
|
| 6266 | return handle_userfault(vmf, VM_UFFD_RWP);
|
| 6267 | }
|
| 6268 |
|
| 6269 | spin_lock(vmf->ptl);
|
| 6270 | if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
|
| 6271 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6272 | return 0;
|
| 6273 | }
|
| 6274 | pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot);
|
| 6275 |
|
| 6276 | pte = pte_clear_uffd(pte);
|
| 6277 | pte = pte_mkyoung(pte);
|
| 6278 | if (!pte_write(pte) &&
|
| 6279 | vma_wants_manual_pte_write_upgrade(vmf->vma) &&
|
| 6280 | can_change_pte_writable(vmf->vma, vmf->address, pte))
|
| 6281 | pte = pte_mkwrite(pte, vmf->vma);
|
| 6282 | set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte);
|
| 6283 | update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
|
| 6284 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6285 | return 0;
|
| 6286 | }
|
| 6287 |
|
| 6288 | static vm_fault_t do_numa_page(struct vm_fault *vmf)
|
| 6289 | {
|
| 6290 | struct vm_area_struct *vma = vmf->vma;
|
| 6291 | struct folio *folio = NULL;
|
| 6292 | int nid = NUMA_NO_NODE;
|
| 6293 | bool writable = false, ignore_writable = false;
|
| 6294 | bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma);
|
| 6295 | int last_cpupid;
|
| 6296 | int target_nid;
|
| 6297 | pte_t pte, old_pte;
|
| 6298 | int flags = 0, nr_pages;
|
| 6299 |
|
| 6300 | |
| 6301 | * The pte cannot be used safely until we verify, while holding the page
|
| 6302 | * table lock, that its contents have not changed during fault handling.
|
| 6303 |
|
| 6304 | spin_lock(vmf->ptl);
|
| 6305 |
|
| 6306 | old_pte = ptep_get(vmf->pte);
|
| 6307 |
|
| 6308 | if (unlikely(!pte_same(old_pte, vmf->orig_pte))) {
|
| 6309 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6310 | return 0;
|
| 6311 | }
|
| 6312 |
|
| 6313 | pte = pte_modify(old_pte, vma->vm_page_prot);
|
| 6314 |
|
| 6315 | |
| 6316 | * Detect now whether the PTE could be writable; this information
|
| 6317 | * is only valid while holding the PT lock.
|
| 6318 |
|
| 6319 | writable = pte_write(pte);
|
| 6320 | if (!writable && pte_write_upgrade &&
|
| 6321 | can_change_pte_writable(vma, vmf->address, pte))
|
| 6322 | writable = true;
|
| 6323 |
|
| 6324 | folio = vm_normal_folio(vma, vmf->address, pte);
|
| 6325 | if (!folio || folio_is_zone_device(folio))
|
| 6326 | goto out_map;
|
| 6327 |
|
| 6328 | nid = folio_nid(folio);
|
| 6329 | nr_pages = folio_nr_pages(folio);
|
| 6330 |
|
| 6331 | target_nid = numa_migrate_check(folio, vmf, vmf->address, &flags,
|
| 6332 | writable, &last_cpupid);
|
| 6333 | if (target_nid == NUMA_NO_NODE)
|
| 6334 | goto out_map;
|
| 6335 | if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) {
|
| 6336 | flags |= TNF_MIGRATE_FAIL;
|
| 6337 | goto out_map;
|
| 6338 | }
|
| 6339 |
|
| 6340 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6341 | writable = false;
|
| 6342 | ignore_writable = true;
|
| 6343 |
|
| 6344 |
|
| 6345 | if (!migrate_misplaced_folio(folio, target_nid)) {
|
| 6346 | nid = target_nid;
|
| 6347 | flags |= TNF_MIGRATED;
|
| 6348 | task_numa_fault(last_cpupid, nid, nr_pages, flags);
|
| 6349 | return 0;
|
| 6350 | }
|
| 6351 |
|
| 6352 | flags |= TNF_MIGRATE_FAIL;
|
| 6353 | vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
|
| 6354 | vmf->address, &vmf->ptl);
|
| 6355 | if (unlikely(!vmf->pte))
|
| 6356 | return 0;
|
| 6357 | if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
|
| 6358 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6359 | return 0;
|
| 6360 | }
|
| 6361 | out_map:
|
| 6362 | |
| 6363 | * Make it present again, depending on how arch implements
|
| 6364 | * non-accessible ptes, some can allow access by kernel mode.
|
| 6365 |
|
| 6366 | if (folio && folio_test_large(folio))
|
| 6367 | numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable,
|
| 6368 | pte_write_upgrade);
|
| 6369 | else
|
| 6370 | numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte,
|
| 6371 | writable);
|
| 6372 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6373 |
|
| 6374 | if (nid != NUMA_NO_NODE)
|
| 6375 | task_numa_fault(last_cpupid, nid, nr_pages, flags);
|
| 6376 | return 0;
|
| 6377 | }
|
| 6378 |
|
| 6379 | static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
|
| 6380 | {
|
| 6381 | struct vm_area_struct *vma = vmf->vma;
|
| 6382 | if (vma_is_anonymous(vma))
|
| 6383 | return do_huge_pmd_anonymous_page(vmf);
|
| 6384 | if (vma->vm_ops->huge_fault)
|
| 6385 | return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
|
| 6386 | return VM_FAULT_FALLBACK;
|
| 6387 | }
|
| 6388 |
|
| 6389 |
|
| 6390 | static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
|
| 6391 | {
|
| 6392 | struct vm_area_struct *vma = vmf->vma;
|
| 6393 | const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
|
| 6394 | vm_fault_t ret;
|
| 6395 |
|
| 6396 | if (vma_is_anonymous(vma)) {
|
| 6397 | if (likely(!unshare) &&
|
| 6398 | userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
|
| 6399 | if (userfaultfd_wp_async(vmf->vma))
|
| 6400 | goto split;
|
| 6401 | return handle_userfault(vmf, VM_UFFD_WP);
|
| 6402 | }
|
| 6403 | return do_huge_pmd_wp_page(vmf);
|
| 6404 | }
|
| 6405 |
|
| 6406 | if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
|
| 6407 | if (vma->vm_ops->huge_fault) {
|
| 6408 | ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
|
| 6409 | if (!(ret & VM_FAULT_FALLBACK))
|
| 6410 | return ret;
|
| 6411 | }
|
| 6412 | }
|
| 6413 |
|
| 6414 | split:
|
| 6415 |
|
| 6416 | __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
|
| 6417 |
|
| 6418 | return VM_FAULT_FALLBACK;
|
| 6419 | }
|
| 6420 |
|
| 6421 | static vm_fault_t create_huge_pud(struct vm_fault *vmf)
|
| 6422 | {
|
| 6423 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
|
| 6424 | defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
|
| 6425 | struct vm_area_struct *vma = vmf->vma;
|
| 6426 |
|
| 6427 | if (vma_is_anonymous(vma))
|
| 6428 | return VM_FAULT_FALLBACK;
|
| 6429 | if (vma->vm_ops->huge_fault)
|
| 6430 | return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
|
| 6431 | #endif
|
| 6432 | return VM_FAULT_FALLBACK;
|
| 6433 | }
|
| 6434 |
|
| 6435 | static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
|
| 6436 | {
|
| 6437 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
|
| 6438 | defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
|
| 6439 | struct vm_area_struct *vma = vmf->vma;
|
| 6440 | vm_fault_t ret;
|
| 6441 |
|
| 6442 |
|
| 6443 | if (vma_is_anonymous(vma))
|
| 6444 | goto split;
|
| 6445 | if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
|
| 6446 | if (vma->vm_ops->huge_fault) {
|
| 6447 | ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
|
| 6448 | if (!(ret & VM_FAULT_FALLBACK))
|
| 6449 | return ret;
|
| 6450 | }
|
| 6451 | }
|
| 6452 | split:
|
| 6453 |
|
| 6454 | __split_huge_pud(vma, vmf->pud, vmf->address);
|
| 6455 | #endif
|
| 6456 | return VM_FAULT_FALLBACK;
|
| 6457 | }
|
| 6458 |
|
| 6459 | |
| 6460 | * The page faults may be spurious because of the racy access to the
|
| 6461 | * page table. For example, a non-populated virtual page is accessed
|
| 6462 | * on 2 CPUs simultaneously, thus the page faults are triggered on
|
| 6463 | * both CPUs. However, it's possible that one CPU (say CPU A) cannot
|
| 6464 | * find the reason for the page fault if the other CPU (say CPU B) has
|
| 6465 | * changed the page table before the PTE is checked on CPU A. Most of
|
| 6466 | * the time, the spurious page faults can be ignored safely. However,
|
| 6467 | * if the page fault is for the write access, it's possible that a
|
| 6468 | * stale read-only TLB entry exists in the local CPU and needs to be
|
| 6469 | * flushed on some architectures. This is called the spurious page
|
| 6470 | * fault fixing.
|
| 6471 | *
|
| 6472 | * Note: flush_tlb_fix_spurious_fault() is defined as flush_tlb_page()
|
| 6473 | * by default and used as such on most architectures, while
|
| 6474 | * flush_tlb_fix_spurious_fault_pmd() is defined as NOP by default and
|
| 6475 | * used as such on most architectures.
|
| 6476 |
|
| 6477 | static void fix_spurious_fault(struct vm_fault *vmf,
|
| 6478 | enum pgtable_level ptlevel)
|
| 6479 | {
|
| 6480 |
|
| 6481 | if (vmf->flags & FAULT_FLAG_TRIED)
|
| 6482 | return;
|
| 6483 | |
| 6484 | * This is needed only for protection faults but the arch code
|
| 6485 | * is not yet telling us if this is a protection fault or not.
|
| 6486 | * This still avoids useless tlb flushes for .text page faults
|
| 6487 | * with threads.
|
| 6488 |
|
| 6489 | if (vmf->flags & FAULT_FLAG_WRITE) {
|
| 6490 | if (ptlevel == PGTABLE_LEVEL_PTE)
|
| 6491 | flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
|
| 6492 | vmf->pte);
|
| 6493 | else
|
| 6494 | flush_tlb_fix_spurious_fault_pmd(vmf->vma, vmf->address,
|
| 6495 | vmf->pmd);
|
| 6496 | }
|
| 6497 | }
|
| 6498 | |
| 6499 | * These routines also need to handle stuff like marking pages dirty
|
| 6500 | * and/or accessed for architectures that don't do it in hardware (most
|
| 6501 | * RISC architectures). The early dirtying is also good on the i386.
|
| 6502 | *
|
| 6503 | * There is also a hook called "update_mmu_cache()" that architectures
|
| 6504 | * with external mmu caches can use to update those (ie the Sparc or
|
| 6505 | * PowerPC hashed page tables that act as extended TLBs).
|
| 6506 | *
|
| 6507 | * On entry, we hold either the VMA lock or the mmap_lock
|
| 6508 | * (see FAULT_FLAG_VMA_LOCK).
|
| 6509 | *
|
| 6510 | * The mmap_lock or VMA lock may have been released depending on flags
|
| 6511 | * and our return value.
|
| 6512 | * See filemap_fault() and __folio_lock_or_retry().
|
| 6513 |
|
| 6514 | static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
|
| 6515 | {
|
| 6516 | pte_t entry;
|
| 6517 |
|
| 6518 | if (unlikely(pmd_none(*vmf->pmd))) {
|
| 6519 | |
| 6520 | * Leave __pte_alloc() until later: because vm_ops->fault may
|
| 6521 | * want to allocate huge page, and if we expose page table
|
| 6522 | * for an instant, it will be difficult to retract from
|
| 6523 | * concurrent faults and from rmap lookups.
|
| 6524 |
|
| 6525 | vmf->pte = NULL;
|
| 6526 | vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
|
| 6527 | } else {
|
| 6528 | pmd_t dummy_pmdval;
|
| 6529 |
|
| 6530 | |
| 6531 | * A regular pmd is established and it can't morph into a huge
|
| 6532 | * pmd by anon khugepaged, since that takes mmap_lock in write
|
| 6533 | * mode; but shmem or file collapse to THP could still morph
|
| 6534 | * it into a huge pmd: just retry later if so.
|
| 6535 | *
|
| 6536 | * Use the maywrite version to indicate that vmf->pte may be
|
| 6537 | * modified, but since we will use pte_same() to detect the
|
| 6538 | * change of the !pte_none() entry, there is no need to recheck
|
| 6539 | * the pmdval. Here we choose to pass a dummy variable instead
|
| 6540 | * of NULL, which helps new user think about why this place is
|
| 6541 | * special.
|
| 6542 |
|
| 6543 | vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd,
|
| 6544 | vmf->address, &dummy_pmdval,
|
| 6545 | &vmf->ptl);
|
| 6546 | if (unlikely(!vmf->pte))
|
| 6547 | return 0;
|
| 6548 | vmf->orig_pte = ptep_get_lockless(vmf->pte);
|
| 6549 | vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
|
| 6550 |
|
| 6551 | if (pte_none(vmf->orig_pte)) {
|
| 6552 | pte_unmap(vmf->pte);
|
| 6553 | vmf->pte = NULL;
|
| 6554 | }
|
| 6555 | }
|
| 6556 |
|
| 6557 | if (!vmf->pte)
|
| 6558 | return do_pte_missing(vmf);
|
| 6559 |
|
| 6560 | if (!pte_present(vmf->orig_pte))
|
| 6561 | return do_swap_page(vmf);
|
| 6562 |
|
| 6563 | if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) {
|
| 6564 | |
| 6565 | * RWP-protected PTEs are protnone plus the uffd bit. On a
|
| 6566 | * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is
|
| 6567 | * NUMA hinting and must still fall through to do_numa_page().
|
| 6568 |
|
| 6569 | if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte))
|
| 6570 | return do_uffd_rwp(vmf);
|
| 6571 | return do_numa_page(vmf);
|
| 6572 | }
|
| 6573 |
|
| 6574 | spin_lock(vmf->ptl);
|
| 6575 | entry = vmf->orig_pte;
|
| 6576 | if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
|
| 6577 | update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
|
| 6578 | goto unlock;
|
| 6579 | }
|
| 6580 | if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
|
| 6581 | if (!pte_write(entry))
|
| 6582 | return do_wp_page(vmf);
|
| 6583 | else if (likely(vmf->flags & FAULT_FLAG_WRITE))
|
| 6584 | entry = pte_mkdirty(entry);
|
| 6585 | }
|
| 6586 | entry = pte_mkyoung(entry);
|
| 6587 | if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
|
| 6588 | vmf->flags & FAULT_FLAG_WRITE))
|
| 6589 | update_mmu_cache_range(vmf, vmf->vma, vmf->address,
|
| 6590 | vmf->pte, 1);
|
| 6591 | else
|
| 6592 | fix_spurious_fault(vmf, PGTABLE_LEVEL_PTE);
|
| 6593 | unlock:
|
| 6594 | pte_unmap_unlock(vmf->pte, vmf->ptl);
|
| 6595 | return 0;
|
| 6596 | }
|
| 6597 |
|
| 6598 | |
| 6599 | * On entry, we hold either the VMA lock or the mmap_lock
|
| 6600 | * (see FAULT_FLAG_VMA_LOCK). If VM_FAULT_RETRY is set in
|
| 6601 | * the result, the lock is not held on exit. See filemap_fault()
|
| 6602 | * and __folio_lock_or_retry().
|
| 6603 |
|
| 6604 | static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
|
| 6605 | unsigned long address, unsigned int flags)
|
| 6606 | {
|
| 6607 | struct vm_fault vmf = {
|
| 6608 | .vma = vma,
|
| 6609 | .address = address & PAGE_MASK,
|
| 6610 | .real_address = address,
|
| 6611 | .flags = flags,
|
| 6612 | .pgoff = linear_page_index(vma, address),
|
| 6613 | .gfp_mask = __get_fault_gfp_mask(vma),
|
| 6614 | };
|
| 6615 | struct mm_struct *mm = vma->vm_mm;
|
| 6616 | vm_flags_t vm_flags = vma->vm_flags;
|
| 6617 | pgd_t *pgd;
|
| 6618 | p4d_t *p4d;
|
| 6619 | vm_fault_t ret;
|
| 6620 |
|
| 6621 | pgd = pgd_offset(mm, address);
|
| 6622 | p4d = p4d_alloc(mm, pgd, address);
|
| 6623 | if (!p4d)
|
| 6624 | return VM_FAULT_OOM;
|
| 6625 |
|
| 6626 | vmf.pud = pud_alloc(mm, p4d, address);
|
| 6627 | if (!vmf.pud)
|
| 6628 | return VM_FAULT_OOM;
|
| 6629 | retry_pud:
|
| 6630 | if (pud_none(*vmf.pud) &&
|
| 6631 | thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PUD_ORDER)) {
|
| 6632 | ret = create_huge_pud(&vmf);
|
| 6633 | if (!(ret & VM_FAULT_FALLBACK))
|
| 6634 | return ret;
|
| 6635 | } else {
|
| 6636 | pud_t orig_pud = *vmf.pud;
|
| 6637 |
|
| 6638 | barrier();
|
| 6639 | if (pud_trans_huge(orig_pud)) {
|
| 6640 |
|
| 6641 | |
| 6642 | * TODO once we support anonymous PUDs: NUMA case and
|
| 6643 | * FAULT_FLAG_UNSHARE handling.
|
| 6644 |
|
| 6645 | if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
|
| 6646 | ret = wp_huge_pud(&vmf, orig_pud);
|
| 6647 | if (!(ret & VM_FAULT_FALLBACK))
|
| 6648 | return ret;
|
| 6649 | } else {
|
| 6650 | huge_pud_set_accessed(&vmf, orig_pud);
|
| 6651 | return 0;
|
| 6652 | }
|
| 6653 | }
|
| 6654 | }
|
| 6655 |
|
| 6656 | vmf.pmd = pmd_alloc(mm, vmf.pud, address);
|
| 6657 | if (!vmf.pmd)
|
| 6658 | return VM_FAULT_OOM;
|
| 6659 |
|
| 6660 |
|
| 6661 | if (pud_trans_unstable(vmf.pud))
|
| 6662 | goto retry_pud;
|
| 6663 |
|
| 6664 | if (pmd_none(*vmf.pmd) &&
|
| 6665 | thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PMD_ORDER)) {
|
| 6666 | ret = create_huge_pmd(&vmf);
|
| 6667 | if (ret & VM_FAULT_FALLBACK)
|
| 6668 | goto fallback;
|
| 6669 | else
|
| 6670 | return ret;
|
| 6671 | }
|
| 6672 |
|
| 6673 | vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
|
| 6674 | if (pmd_none(vmf.orig_pmd))
|
| 6675 | goto fallback;
|
| 6676 |
|
| 6677 | if (unlikely(!pmd_present(vmf.orig_pmd))) {
|
| 6678 | if (pmd_is_device_private_entry(vmf.orig_pmd))
|
| 6679 | return do_huge_pmd_device_private(&vmf);
|
| 6680 |
|
| 6681 | if (pmd_is_migration_entry(vmf.orig_pmd))
|
| 6682 | pmd_migration_entry_wait(mm, vmf.pmd);
|
| 6683 | return 0;
|
| 6684 | }
|
| 6685 | if (pmd_trans_huge(vmf.orig_pmd)) {
|
| 6686 | if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) {
|
| 6687 | if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd))
|
| 6688 | return do_huge_pmd_uffd_rwp(&vmf);
|
| 6689 | return do_huge_pmd_numa_page(&vmf);
|
| 6690 | }
|
| 6691 |
|
| 6692 | if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
|
| 6693 | !pmd_write(vmf.orig_pmd)) {
|
| 6694 | ret = wp_huge_pmd(&vmf);
|
| 6695 | if (!(ret & VM_FAULT_FALLBACK))
|
| 6696 | return ret;
|
| 6697 | } else {
|
| 6698 | vmf.ptl = pmd_lock(mm, vmf.pmd);
|
| 6699 | if (!huge_pmd_set_accessed(&vmf))
|
| 6700 | fix_spurious_fault(&vmf, PGTABLE_LEVEL_PMD);
|
| 6701 | spin_unlock(vmf.ptl);
|
| 6702 | return 0;
|
| 6703 | }
|
| 6704 | }
|
| 6705 |
|
| 6706 | fallback:
|
| 6707 | return handle_pte_fault(&vmf);
|
| 6708 | }
|
| 6709 |
|
| 6710 | |
| 6711 | * mm_account_fault - Do page fault accounting
|
| 6712 | * @mm: mm from which memcg should be extracted. It can be NULL.
|
| 6713 | * @regs: the pt_regs struct pointer. When set to NULL, will skip accounting
|
| 6714 | * of perf event counters, but we'll still do the per-task accounting to
|
| 6715 | * the task who triggered this page fault.
|
| 6716 | * @address: the faulted address.
|
| 6717 | * @flags: the fault flags.
|
| 6718 | * @ret: the fault retcode.
|
| 6719 | *
|
| 6720 | * This will take care of most of the page fault accounting. Meanwhile, it
|
| 6721 | * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
|
| 6722 | * updates. However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
|
| 6723 | * still be in per-arch page fault handlers at the entry of page fault.
|
| 6724 |
|
| 6725 | static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
|
| 6726 | unsigned long address, unsigned int flags,
|
| 6727 | vm_fault_t ret)
|
| 6728 | {
|
| 6729 | bool major;
|
| 6730 |
|
| 6731 |
|
| 6732 | if (ret & VM_FAULT_RETRY)
|
| 6733 | return;
|
| 6734 |
|
| 6735 | |
| 6736 | * To preserve the behavior of older kernels, PGFAULT counters record
|
| 6737 | * both successful and failed faults, as opposed to perf counters,
|
| 6738 | * which ignore failed cases.
|
| 6739 |
|
| 6740 | count_vm_event(PGFAULT);
|
| 6741 | count_memcg_event_mm(mm, PGFAULT);
|
| 6742 |
|
| 6743 | |
| 6744 | * Do not account for unsuccessful faults (e.g. when the address wasn't
|
| 6745 | * valid). That includes arch_vma_access_permitted() failing before
|
| 6746 | * reaching here. So this is not a "this many hardware page faults"
|
| 6747 | * counter. We should use the hw profiling for that.
|
| 6748 |
|
| 6749 | if (ret & VM_FAULT_ERROR)
|
| 6750 | return;
|
| 6751 |
|
| 6752 | |
| 6753 | * We define the fault as a major fault when the final successful fault
|
| 6754 | * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
|
| 6755 | * handle it immediately previously).
|
| 6756 |
|
| 6757 | major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
|
| 6758 |
|
| 6759 | if (major)
|
| 6760 | current->maj_flt++;
|
| 6761 | else
|
| 6762 | current->min_flt++;
|
| 6763 |
|
| 6764 | |
| 6765 | * If the fault is done for GUP, regs will be NULL. We only do the
|
| 6766 | * accounting for the per thread fault counters who triggered the
|
| 6767 | * fault, and we skip the perf event updates.
|
| 6768 |
|
| 6769 | if (!regs)
|
| 6770 | return;
|
| 6771 |
|
| 6772 | if (major)
|
| 6773 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
|
| 6774 | else
|
| 6775 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
|
| 6776 | }
|
| 6777 |
|
| 6778 | #ifdef CONFIG_LRU_GEN
|
| 6779 | static void lru_gen_enter_fault(struct vm_area_struct *vma)
|
| 6780 | {
|
| 6781 |
|
| 6782 | current->in_lru_fault = vma_has_recency(vma);
|
| 6783 | }
|
| 6784 |
|
| 6785 | static void lru_gen_exit_fault(void)
|
| 6786 | {
|
| 6787 | current->in_lru_fault = false;
|
| 6788 | }
|
| 6789 | #else
|
| 6790 | static void lru_gen_enter_fault(struct vm_area_struct *vma)
|
| 6791 | {
|
| 6792 | }
|
| 6793 |
|
| 6794 | static void lru_gen_exit_fault(void)
|
| 6795 | {
|
| 6796 | }
|
| 6797 | #endif
|
| 6798 |
|
| 6799 | static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
|
| 6800 | unsigned int *flags)
|
| 6801 | {
|
| 6802 | if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
|
| 6803 | if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
|
| 6804 | return VM_FAULT_SIGSEGV;
|
| 6805 | |
| 6806 | * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
|
| 6807 | * just treat it like an ordinary read-fault otherwise.
|
| 6808 |
|
| 6809 | if (!vma_is_cow_mapping(vma))
|
| 6810 | *flags &= ~FAULT_FLAG_UNSHARE;
|
| 6811 | } else if (*flags & FAULT_FLAG_WRITE) {
|
| 6812 |
|
| 6813 | if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
|
| 6814 | return VM_FAULT_SIGSEGV;
|
| 6815 |
|
| 6816 | if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
|
| 6817 | !vma_is_cow_mapping(vma)))
|
| 6818 | return VM_FAULT_SIGSEGV;
|
| 6819 | }
|
| 6820 | #ifdef CONFIG_PER_VMA_LOCK
|
| 6821 | |
| 6822 | * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
|
| 6823 | * the assumption that lock is dropped on VM_FAULT_RETRY.
|
| 6824 |
|
| 6825 | if (WARN_ON_ONCE((*flags &
|
| 6826 | (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
|
| 6827 | (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
|
| 6828 | return VM_FAULT_SIGSEGV;
|
| 6829 | #endif
|
| 6830 |
|
| 6831 | return 0;
|
| 6832 | }
|
| 6833 |
|
| 6834 | |
| 6835 | * By the time we get here, we already hold either the VMA lock or the
|
| 6836 | * mmap_lock (see FAULT_FLAG_VMA_LOCK).
|
| 6837 | *
|
| 6838 | * The lock may have been released depending on flags and our
|
| 6839 | * return value. See filemap_fault() and __folio_lock_or_retry().
|
| 6840 |
|
| 6841 | vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
|
| 6842 | unsigned int flags, struct pt_regs *regs)
|
| 6843 | {
|
| 6844 |
|
| 6845 | struct mm_struct *mm = vma->vm_mm;
|
| 6846 | vm_fault_t ret;
|
| 6847 | bool is_droppable;
|
| 6848 |
|
| 6849 | __set_current_state(TASK_RUNNING);
|
| 6850 |
|
| 6851 | ret = sanitize_fault_flags(vma, &flags);
|
| 6852 | if (ret)
|
| 6853 | goto out;
|
| 6854 |
|
| 6855 | if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
|
| 6856 | flags & FAULT_FLAG_INSTRUCTION,
|
| 6857 | flags & FAULT_FLAG_REMOTE)) {
|
| 6858 | ret = VM_FAULT_SIGSEGV;
|
| 6859 | goto out;
|
| 6860 | }
|
| 6861 |
|
| 6862 | is_droppable = !!(vma->vm_flags & VM_DROPPABLE);
|
| 6863 |
|
| 6864 | |
| 6865 | * Enable the memcg OOM handling for faults triggered in user
|
| 6866 | * space. Kernel faults are handled more gracefully.
|
| 6867 |
|
| 6868 | if (flags & FAULT_FLAG_USER)
|
| 6869 | mem_cgroup_enter_user_fault();
|
| 6870 |
|
| 6871 | lru_gen_enter_fault(vma);
|
| 6872 |
|
| 6873 | if (unlikely(is_vm_hugetlb_page(vma)))
|
| 6874 | ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
|
| 6875 | else
|
| 6876 | ret = __handle_mm_fault(vma, address, flags);
|
| 6877 |
|
| 6878 | |
| 6879 | * Warning: It is no longer safe to dereference vma-> after this point,
|
| 6880 | * because mmap_lock might have been dropped by __handle_mm_fault(), so
|
| 6881 | * vma might be destroyed from underneath us.
|
| 6882 |
|
| 6883 |
|
| 6884 | lru_gen_exit_fault();
|
| 6885 |
|
| 6886 |
|
| 6887 | if (is_droppable)
|
| 6888 | ret &= ~VM_FAULT_OOM;
|
| 6889 |
|
| 6890 | if (flags & FAULT_FLAG_USER) {
|
| 6891 | mem_cgroup_exit_user_fault();
|
| 6892 | |
| 6893 | * The task may have entered a memcg OOM situation but
|
| 6894 | * if the allocation error was handled gracefully (no
|
| 6895 | * VM_FAULT_OOM), there is no need to kill anything.
|
| 6896 | * Just clean up the OOM state peacefully.
|
| 6897 |
|
| 6898 | if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
|
| 6899 | mem_cgroup_oom_synchronize(false);
|
| 6900 | }
|
| 6901 | out:
|
| 6902 | mm_account_fault(mm, regs, address, flags, ret);
|
| 6903 |
|
| 6904 | return ret;
|
| 6905 | }
|
| 6906 | EXPORT_SYMBOL_GPL(handle_mm_fault);
|
| 6907 |
|
| 6908 | #ifndef __PAGETABLE_P4D_FOLDED
|
| 6909 | |
| 6910 | * Allocate p4d page table.
|
| 6911 | * We've already handled the fast-path in-line.
|
| 6912 |
|
| 6913 | int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
|
| 6914 | {
|
| 6915 | p4d_t *new = p4d_alloc_one(mm, address);
|
| 6916 | if (!new)
|
| 6917 | return -ENOMEM;
|
| 6918 |
|
| 6919 | spin_lock(&mm->page_table_lock);
|
| 6920 | if (pgd_present(*pgd)) {
|
| 6921 | p4d_free(mm, new);
|
| 6922 | } else {
|
| 6923 | smp_wmb();
|
| 6924 | pgd_populate(mm, pgd, new);
|
| 6925 | }
|
| 6926 | spin_unlock(&mm->page_table_lock);
|
| 6927 | return 0;
|
| 6928 | }
|
| 6929 | #endif
|
| 6930 |
|
| 6931 | #ifndef __PAGETABLE_PUD_FOLDED
|
| 6932 | |
| 6933 | * Allocate page upper directory.
|
| 6934 | * We've already handled the fast-path in-line.
|
| 6935 |
|
| 6936 | int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
|
| 6937 | {
|
| 6938 | pud_t *new = pud_alloc_one(mm, address);
|
| 6939 | if (!new)
|
| 6940 | return -ENOMEM;
|
| 6941 |
|
| 6942 | spin_lock(&mm->page_table_lock);
|
| 6943 | if (!p4d_present(*p4d)) {
|
| 6944 | mm_inc_nr_puds(mm);
|
| 6945 | smp_wmb();
|
| 6946 | p4d_populate(mm, p4d, new);
|
| 6947 | } else
|
| 6948 | pud_free(mm, new);
|
| 6949 | spin_unlock(&mm->page_table_lock);
|
| 6950 | return 0;
|
| 6951 | }
|
| 6952 | #endif
|
| 6953 |
|
| 6954 | #ifndef __PAGETABLE_PMD_FOLDED
|
| 6955 | |
| 6956 | * Allocate page middle directory.
|
| 6957 | * We've already handled the fast-path in-line.
|
| 6958 |
|
| 6959 | int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
|
| 6960 | {
|
| 6961 | spinlock_t *ptl;
|
| 6962 | pmd_t *new = pmd_alloc_one(mm, address);
|
| 6963 | if (!new)
|
| 6964 | return -ENOMEM;
|
| 6965 |
|
| 6966 | ptl = pud_lock(mm, pud);
|
| 6967 | if (!pud_present(*pud)) {
|
| 6968 | mm_inc_nr_pmds(mm);
|
| 6969 | smp_wmb();
|
| 6970 | pud_populate(mm, pud, new);
|
| 6971 | } else {
|
| 6972 | pmd_free(mm, new);
|
| 6973 | }
|
| 6974 | spin_unlock(ptl);
|
| 6975 | return 0;
|
| 6976 | }
|
| 6977 | #endif
|
| 6978 |
|
| 6979 | static inline void pfnmap_args_setup(struct follow_pfnmap_args *args,
|
| 6980 | spinlock_t *lock, pte_t *ptep,
|
| 6981 | pgprot_t pgprot, unsigned long pfn_base,
|
| 6982 | unsigned long addr_mask, bool writable,
|
| 6983 | bool special)
|
| 6984 | {
|
| 6985 | args->lock = lock;
|
| 6986 | args->ptep = ptep;
|
| 6987 | args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT);
|
| 6988 | args->addr_mask = addr_mask;
|
| 6989 | args->pgprot = pgprot;
|
| 6990 | args->writable = writable;
|
| 6991 | args->special = special;
|
| 6992 | }
|
| 6993 |
|
| 6994 | static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma)
|
| 6995 | {
|
| 6996 | #ifdef CONFIG_LOCKDEP
|
| 6997 | struct file *file = vma->vm_file;
|
| 6998 | struct address_space *mapping = file ? file->f_mapping : NULL;
|
| 6999 |
|
| 7000 | if (mapping)
|
| 7001 | lockdep_assert(lockdep_is_held(&mapping->i_mmap_rwsem) ||
|
| 7002 | lockdep_is_held(&vma->vm_mm->mmap_lock));
|
| 7003 | else
|
| 7004 | lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));
|
| 7005 | #endif
|
| 7006 | }
|
| 7007 |
|
| 7008 | |
| 7009 | * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address
|
| 7010 | * @args: Pointer to struct @follow_pfnmap_args
|
| 7011 | *
|
| 7012 | * The caller needs to setup args->vma and args->address to point to the
|
| 7013 | * virtual address as the target of such lookup. On a successful return,
|
| 7014 | * the results will be put into other output fields.
|
| 7015 | *
|
| 7016 | * After the caller finished using the fields, the caller must invoke
|
| 7017 | * another follow_pfnmap_end() to proper releases the locks and resources
|
| 7018 | * of such look up request.
|
| 7019 | *
|
| 7020 | * During the start() and end() calls, the results in @args will be valid
|
| 7021 | * as proper locks will be held. After the end() is called, all the fields
|
| 7022 | * in @follow_pfnmap_args will be invalid to be further accessed. Further
|
| 7023 | * use of such information after end() may require proper synchronizations
|
| 7024 | * by the caller with page table updates, otherwise it can create a
|
| 7025 | * security bug.
|
| 7026 | *
|
| 7027 | * If the PTE maps a refcounted page, callers are responsible to protect
|
| 7028 | * against invalidation with MMU notifiers; otherwise access to the PFN at
|
| 7029 | * a later point in time can trigger use-after-free.
|
| 7030 | *
|
| 7031 | * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore
|
| 7032 | * should be taken for read, and the mmap semaphore cannot be released
|
| 7033 | * before the end() is invoked.
|
| 7034 | *
|
| 7035 | * This function must not be used to modify PTE content.
|
| 7036 | *
|
| 7037 | * Return: zero on success, negative otherwise.
|
| 7038 |
|
| 7039 | int follow_pfnmap_start(struct follow_pfnmap_args *args)
|
| 7040 | {
|
| 7041 | struct vm_area_struct *vma = args->vma;
|
| 7042 | unsigned long address = args->address;
|
| 7043 | struct mm_struct *mm = vma->vm_mm;
|
| 7044 | spinlock_t *lock;
|
| 7045 | pgd_t *pgdp;
|
| 7046 | p4d_t *p4dp, p4d;
|
| 7047 | pud_t *pudp, pud;
|
| 7048 | pmd_t *pmdp, pmd;
|
| 7049 | pte_t *ptep, pte;
|
| 7050 |
|
| 7051 | pfnmap_lockdep_assert(vma);
|
| 7052 |
|
| 7053 | if (unlikely(address < vma->vm_start || address >= vma->vm_end))
|
| 7054 | goto out;
|
| 7055 |
|
| 7056 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
|
| 7057 | goto out;
|
| 7058 | retry:
|
| 7059 | pgdp = pgd_offset(mm, address);
|
| 7060 | if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
|
| 7061 | goto out;
|
| 7062 |
|
| 7063 | p4dp = p4d_offset(pgdp, address);
|
| 7064 | p4d = p4dp_get(p4dp);
|
| 7065 | if (p4d_none(p4d) || unlikely(p4d_bad(p4d)))
|
| 7066 | goto out;
|
| 7067 |
|
| 7068 | pudp = pud_offset(p4dp, address);
|
| 7069 | pud = pudp_get(pudp);
|
| 7070 | if (!pud_present(pud))
|
| 7071 | goto out;
|
| 7072 | if (pud_leaf(pud)) {
|
| 7073 | lock = pud_lock(mm, pudp);
|
| 7074 | pud = pudp_get(pudp);
|
| 7075 |
|
| 7076 | if (unlikely(!pud_present(pud))) {
|
| 7077 | spin_unlock(lock);
|
| 7078 | goto out;
|
| 7079 | } else if (unlikely(!pud_leaf(pud))) {
|
| 7080 | spin_unlock(lock);
|
| 7081 | goto retry;
|
| 7082 | }
|
| 7083 | pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud),
|
| 7084 | pud_pfn(pud), PUD_MASK, pud_write(pud),
|
| 7085 | pud_special(pud));
|
| 7086 | return 0;
|
| 7087 | }
|
| 7088 |
|
| 7089 | pmdp = pmd_offset(pudp, address);
|
| 7090 | pmd = pmdp_get_lockless(pmdp);
|
| 7091 | if (!pmd_present(pmd))
|
| 7092 | goto out;
|
| 7093 | if (pmd_leaf(pmd)) {
|
| 7094 | lock = pmd_lock(mm, pmdp);
|
| 7095 | pmd = pmdp_get(pmdp);
|
| 7096 |
|
| 7097 | if (unlikely(!pmd_present(pmd))) {
|
| 7098 | spin_unlock(lock);
|
| 7099 | goto out;
|
| 7100 | } else if (unlikely(!pmd_leaf(pmd))) {
|
| 7101 | spin_unlock(lock);
|
| 7102 | goto retry;
|
| 7103 | }
|
| 7104 | pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd),
|
| 7105 | pmd_pfn(pmd), PMD_MASK, pmd_write(pmd),
|
| 7106 | pmd_special(pmd));
|
| 7107 | return 0;
|
| 7108 | }
|
| 7109 |
|
| 7110 | ptep = pte_offset_map_lock(mm, pmdp, address, &lock);
|
| 7111 | if (!ptep)
|
| 7112 | goto out;
|
| 7113 | pte = ptep_get(ptep);
|
| 7114 | if (!pte_present(pte))
|
| 7115 | goto unlock;
|
| 7116 | pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte),
|
| 7117 | pte_pfn(pte), PAGE_MASK, pte_write(pte),
|
| 7118 | pte_special(pte));
|
| 7119 | return 0;
|
| 7120 | unlock:
|
| 7121 | pte_unmap_unlock(ptep, lock);
|
| 7122 | out:
|
| 7123 | return -EINVAL;
|
| 7124 | }
|
| 7125 | EXPORT_SYMBOL_GPL(follow_pfnmap_start);
|
| 7126 |
|
| 7127 | |
| 7128 | * follow_pfnmap_end(): End a follow_pfnmap_start() process
|
| 7129 | * @args: Pointer to struct @follow_pfnmap_args
|
| 7130 | *
|
| 7131 | * Must be used in pair of follow_pfnmap_start(). See the start() function
|
| 7132 | * above for more information.
|
| 7133 |
|
| 7134 | void follow_pfnmap_end(struct follow_pfnmap_args *args)
|
| 7135 | {
|
| 7136 | if (args->lock)
|
| 7137 | spin_unlock(args->lock);
|
| 7138 | if (args->ptep)
|
| 7139 | pte_unmap(args->ptep);
|
| 7140 | }
|
| 7141 | EXPORT_SYMBOL_GPL(follow_pfnmap_end);
|
| 7142 |
|
| 7143 | #ifdef CONFIG_HAVE_IOREMAP_PROT
|
| 7144 | |
| 7145 | * generic_access_phys - generic implementation for iomem mmap access
|
| 7146 | * @vma: the vma to access
|
| 7147 | * @addr: userspace address, not relative offset within @vma
|
| 7148 | * @buf: buffer to read/write
|
| 7149 | * @len: length of transfer
|
| 7150 | * @write: set to FOLL_WRITE when writing, otherwise reading
|
| 7151 | *
|
| 7152 | * This is a generic implementation for &vm_operations_struct.access for an
|
| 7153 | * iomem mapping. This callback is used by access_process_vm() when the @vma is
|
| 7154 | * not page based.
|
| 7155 |
|
| 7156 | int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
|
| 7157 | void *buf, int len, int write)
|
| 7158 | {
|
| 7159 | resource_size_t phys_addr;
|
| 7160 | pgprot_t prot = __pgprot(0);
|
| 7161 | void __iomem *maddr;
|
| 7162 | int offset = offset_in_page(addr);
|
| 7163 | int ret = -EINVAL;
|
| 7164 | bool writable;
|
| 7165 | struct follow_pfnmap_args args = { .vma = vma, .address = addr };
|
| 7166 |
|
| 7167 | retry:
|
| 7168 | if (follow_pfnmap_start(&args))
|
| 7169 | return -EINVAL;
|
| 7170 | prot = args.pgprot;
|
| 7171 | phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT;
|
| 7172 | writable = args.writable;
|
| 7173 | follow_pfnmap_end(&args);
|
| 7174 |
|
| 7175 | if ((write & FOLL_WRITE) && !writable)
|
| 7176 | return -EINVAL;
|
| 7177 |
|
| 7178 | maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
|
| 7179 | if (!maddr)
|
| 7180 | return -ENOMEM;
|
| 7181 |
|
| 7182 | if (follow_pfnmap_start(&args))
|
| 7183 | goto out_unmap;
|
| 7184 |
|
| 7185 | if ((pgprot_val(prot) != pgprot_val(args.pgprot)) ||
|
| 7186 | (phys_addr != (args.pfn << PAGE_SHIFT)) ||
|
| 7187 | (writable != args.writable)) {
|
| 7188 | follow_pfnmap_end(&args);
|
| 7189 | iounmap(maddr);
|
| 7190 | goto retry;
|
| 7191 | }
|
| 7192 |
|
| 7193 | if (write)
|
| 7194 | memcpy_toio(maddr + offset, buf, len);
|
| 7195 | else
|
| 7196 | memcpy_fromio(buf, maddr + offset, len);
|
| 7197 | ret = len;
|
| 7198 | follow_pfnmap_end(&args);
|
| 7199 | out_unmap:
|
| 7200 | iounmap(maddr);
|
| 7201 |
|
| 7202 | return ret;
|
| 7203 | }
|
| 7204 | EXPORT_SYMBOL_GPL(generic_access_phys);
|
| 7205 | #endif
|
| 7206 |
|
| 7207 | |
| 7208 | * Access another process' address space as given in mm.
|
| 7209 |
|
| 7210 | static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
|
| 7211 | void *buf, int len, unsigned int gup_flags)
|
| 7212 | {
|
| 7213 | void *old_buf = buf;
|
| 7214 | int write = gup_flags & FOLL_WRITE;
|
| 7215 |
|
| 7216 | if (mmap_read_lock_killable(mm))
|
| 7217 | return 0;
|
| 7218 |
|
| 7219 |
|
| 7220 | addr = untagged_addr_remote(mm, addr);
|
| 7221 |
|
| 7222 |
|
| 7223 | if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
|
| 7224 | return 0;
|
| 7225 |
|
| 7226 |
|
| 7227 | while (len) {
|
| 7228 | int bytes, offset;
|
| 7229 | void *maddr;
|
| 7230 | struct folio *folio;
|
| 7231 | struct vm_area_struct *vma = NULL;
|
| 7232 | struct page *page = get_user_page_vma_remote(mm, addr,
|
| 7233 | gup_flags, &vma);
|
| 7234 |
|
| 7235 | if (IS_ERR(page)) {
|
| 7236 |
|
| 7237 | vma = vma_lookup(mm, addr);
|
| 7238 | if (!vma) {
|
| 7239 | vma = expand_stack(mm, addr);
|
| 7240 |
|
| 7241 |
|
| 7242 | if (!vma)
|
| 7243 | return buf - old_buf;
|
| 7244 |
|
| 7245 |
|
| 7246 | continue;
|
| 7247 | }
|
| 7248 |
|
| 7249 | |
| 7250 | * Check if this is a VM_IO | VM_PFNMAP VMA, which
|
| 7251 | * we can access using slightly different code.
|
| 7252 |
|
| 7253 | bytes = 0;
|
| 7254 | #ifdef CONFIG_HAVE_IOREMAP_PROT
|
| 7255 | if (vma->vm_ops && vma->vm_ops->access)
|
| 7256 | bytes = vma->vm_ops->access(vma, addr, buf,
|
| 7257 | len, write);
|
| 7258 | #endif
|
| 7259 | if (bytes <= 0)
|
| 7260 | break;
|
| 7261 | } else {
|
| 7262 | folio = page_folio(page);
|
| 7263 | bytes = len;
|
| 7264 | offset = addr & (PAGE_SIZE-1);
|
| 7265 | if (bytes > PAGE_SIZE-offset)
|
| 7266 | bytes = PAGE_SIZE-offset;
|
| 7267 |
|
| 7268 | maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE);
|
| 7269 | if (write) {
|
| 7270 | copy_to_user_page(vma, page, addr,
|
| 7271 | maddr + offset, buf, bytes);
|
| 7272 | folio_mark_dirty_lock(folio);
|
| 7273 | } else {
|
| 7274 | copy_from_user_page(vma, page, addr,
|
| 7275 | buf, maddr + offset, bytes);
|
| 7276 | }
|
| 7277 | folio_release_kmap(folio, maddr);
|
| 7278 | }
|
| 7279 | len -= bytes;
|
| 7280 | buf += bytes;
|
| 7281 | addr += bytes;
|
| 7282 | }
|
| 7283 | mmap_read_unlock(mm);
|
| 7284 |
|
| 7285 | return buf - old_buf;
|
| 7286 | }
|
| 7287 |
|
| 7288 | |
| 7289 | * access_remote_vm - access another process' address space
|
| 7290 | * @mm: the mm_struct of the target address space
|
| 7291 | * @addr: start address to access
|
| 7292 | * @buf: source or destination buffer
|
| 7293 | * @len: number of bytes to transfer
|
| 7294 | * @gup_flags: flags modifying lookup behaviour
|
| 7295 | *
|
| 7296 | * The caller must hold a reference on @mm.
|
| 7297 | *
|
| 7298 | * Return: number of bytes copied from source to destination.
|
| 7299 |
|
| 7300 | int access_remote_vm(struct mm_struct *mm, unsigned long addr,
|
| 7301 | void *buf, int len, unsigned int gup_flags)
|
| 7302 | {
|
| 7303 | return __access_remote_vm(mm, addr, buf, len, gup_flags);
|
| 7304 | }
|
| 7305 |
|
| 7306 | |
| 7307 | * Access another process' address space.
|
| 7308 | * Source/target buffer must be kernel space,
|
| 7309 | * Do not walk the page table directly, use get_user_pages
|
| 7310 |
|
| 7311 | int access_process_vm(struct task_struct *tsk, unsigned long addr,
|
| 7312 | void *buf, int len, unsigned int gup_flags)
|
| 7313 | {
|
| 7314 | struct mm_struct *mm;
|
| 7315 | int ret;
|
| 7316 |
|
| 7317 | mm = get_task_mm(tsk);
|
| 7318 | if (!mm)
|
| 7319 | return 0;
|
| 7320 |
|
| 7321 | ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
|
| 7322 |
|
| 7323 | mmput(mm);
|
| 7324 |
|
| 7325 | return ret;
|
| 7326 | }
|
| 7327 | EXPORT_SYMBOL_GPL(access_process_vm);
|
| 7328 |
|
| 7329 | #ifdef CONFIG_BPF_SYSCALL
|
| 7330 | |
| 7331 | * Copy a string from another process's address space as given in mm.
|
| 7332 | * If there is any error return -EFAULT.
|
| 7333 |
|
| 7334 | static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
|
| 7335 | void *buf, int len, unsigned int gup_flags)
|
| 7336 | {
|
| 7337 | void *old_buf = buf;
|
| 7338 | int err = 0;
|
| 7339 |
|
| 7340 | *(char *)buf = '\0';
|
| 7341 |
|
| 7342 | if (mmap_read_lock_killable(mm))
|
| 7343 | return -EFAULT;
|
| 7344 |
|
| 7345 | addr = untagged_addr_remote(mm, addr);
|
| 7346 |
|
| 7347 |
|
| 7348 | if (!vma_lookup(mm, addr)) {
|
| 7349 | err = -EFAULT;
|
| 7350 | goto out;
|
| 7351 | }
|
| 7352 |
|
| 7353 | while (len) {
|
| 7354 | int bytes, offset, retval;
|
| 7355 | void *maddr;
|
| 7356 | struct folio *folio;
|
| 7357 | struct page *page;
|
| 7358 | struct vm_area_struct *vma = NULL;
|
| 7359 |
|
| 7360 | page = get_user_page_vma_remote(mm, addr, gup_flags, &vma);
|
| 7361 | if (IS_ERR(page)) {
|
| 7362 | |
| 7363 | * Treat as a total failure for now until we decide how
|
| 7364 | * to handle the CONFIG_HAVE_IOREMAP_PROT case and
|
| 7365 | * stack expansion.
|
| 7366 |
|
| 7367 | *(char *)buf = '\0';
|
| 7368 | err = -EFAULT;
|
| 7369 | goto out;
|
| 7370 | }
|
| 7371 |
|
| 7372 | folio = page_folio(page);
|
| 7373 | bytes = len;
|
| 7374 | offset = addr & (PAGE_SIZE - 1);
|
| 7375 | if (bytes > PAGE_SIZE - offset)
|
| 7376 | bytes = PAGE_SIZE - offset;
|
| 7377 |
|
| 7378 | maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE);
|
| 7379 | retval = strscpy(buf, maddr + offset, bytes);
|
| 7380 | if (retval >= 0) {
|
| 7381 |
|
| 7382 | buf += retval;
|
| 7383 | folio_release_kmap(folio, maddr);
|
| 7384 | break;
|
| 7385 | }
|
| 7386 |
|
| 7387 | buf += bytes - 1;
|
| 7388 | |
| 7389 | * Because strscpy always NUL terminates we need to
|
| 7390 | * copy the last byte in the page if we are going to
|
| 7391 | * load more pages
|
| 7392 |
|
| 7393 | if (bytes != len) {
|
| 7394 | addr += bytes - 1;
|
| 7395 | copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1);
|
| 7396 | buf += 1;
|
| 7397 | addr += 1;
|
| 7398 | }
|
| 7399 | len -= bytes;
|
| 7400 |
|
| 7401 | folio_release_kmap(folio, maddr);
|
| 7402 | }
|
| 7403 |
|
| 7404 | out:
|
| 7405 | mmap_read_unlock(mm);
|
| 7406 | if (err)
|
| 7407 | return err;
|
| 7408 | return buf - old_buf;
|
| 7409 | }
|
| 7410 |
|
| 7411 | |
| 7412 | * copy_remote_vm_str - copy a string from another process's address space.
|
| 7413 | * @tsk: the task of the target address space
|
| 7414 | * @addr: start address to read from
|
| 7415 | * @buf: destination buffer
|
| 7416 | * @len: number of bytes to copy
|
| 7417 | * @gup_flags: flags modifying lookup behaviour
|
| 7418 | *
|
| 7419 | * The caller must hold a reference on @mm.
|
| 7420 | *
|
| 7421 | * Return: number of bytes copied from @addr (source) to @buf (destination);
|
| 7422 | * not including the trailing NUL. Always guaranteed to leave NUL-terminated
|
| 7423 | * buffer. On any error, return -EFAULT.
|
| 7424 |
|
| 7425 | int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
|
| 7426 | void *buf, int len, unsigned int gup_flags)
|
| 7427 | {
|
| 7428 | struct mm_struct *mm;
|
| 7429 | int ret;
|
| 7430 |
|
| 7431 | if (unlikely(len == 0))
|
| 7432 | return 0;
|
| 7433 |
|
| 7434 | mm = get_task_mm(tsk);
|
| 7435 | if (!mm) {
|
| 7436 | *(char *)buf = '\0';
|
| 7437 | return -EFAULT;
|
| 7438 | }
|
| 7439 |
|
| 7440 | ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags);
|
| 7441 |
|
| 7442 | mmput(mm);
|
| 7443 |
|
| 7444 | return ret;
|
| 7445 | }
|
| 7446 | EXPORT_SYMBOL_GPL(copy_remote_vm_str);
|
| 7447 | #endif
|
| 7448 |
|
| 7449 | |
| 7450 | * Print the name of a VMA.
|
| 7451 |
|
| 7452 | void print_vma_addr(char *prefix, unsigned long ip)
|
| 7453 | {
|
| 7454 | struct mm_struct *mm = current->mm;
|
| 7455 | struct vm_area_struct *vma;
|
| 7456 |
|
| 7457 | |
| 7458 | * we might be running from an atomic context so we cannot sleep
|
| 7459 |
|
| 7460 | if (!mmap_read_trylock(mm))
|
| 7461 | return;
|
| 7462 |
|
| 7463 | vma = vma_lookup(mm, ip);
|
| 7464 | if (vma && vma->vm_file) {
|
| 7465 | struct file *f = vma->vm_file;
|
| 7466 | ip -= vma->vm_start;
|
| 7467 | ip += vma_start_pgoff(vma) << PAGE_SHIFT;
|
| 7468 | printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip,
|
| 7469 | vma->vm_start,
|
| 7470 | vma->vm_end - vma->vm_start);
|
| 7471 | }
|
| 7472 | mmap_read_unlock(mm);
|
| 7473 | }
|
| 7474 |
|
| 7475 | #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
|
| 7476 | void __might_fault(const char *file, int line)
|
| 7477 | {
|
| 7478 | if (pagefault_disabled())
|
| 7479 | return;
|
| 7480 | __might_sleep(file, line);
|
| 7481 | if (current->mm)
|
| 7482 | might_lock_read(¤t->mm->mmap_lock);
|
| 7483 | }
|
| 7484 | EXPORT_SYMBOL(__might_fault);
|
| 7485 | #endif
|
| 7486 |
|
| 7487 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
|
| 7488 | |
| 7489 | * Process all subpages of the specified huge page with the specified
|
| 7490 | * operation. The target subpage will be processed last to keep its
|
| 7491 | * cache lines hot.
|
| 7492 |
|
| 7493 | static inline int process_huge_page(
|
| 7494 | unsigned long addr_hint, unsigned int nr_pages,
|
| 7495 | int (*process_subpage)(unsigned long addr, int idx, void *arg),
|
| 7496 | void *arg)
|
| 7497 | {
|
| 7498 | int i, n, base, l, ret;
|
| 7499 | unsigned long addr = addr_hint &
|
| 7500 | ~(((unsigned long)nr_pages << PAGE_SHIFT) - 1);
|
| 7501 |
|
| 7502 |
|
| 7503 | might_sleep();
|
| 7504 | n = (addr_hint - addr) / PAGE_SIZE;
|
| 7505 | if (2 * n <= nr_pages) {
|
| 7506 |
|
| 7507 | base = 0;
|
| 7508 | l = n;
|
| 7509 |
|
| 7510 | for (i = nr_pages - 1; i >= 2 * n; i--) {
|
| 7511 | cond_resched();
|
| 7512 | ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
|
| 7513 | if (ret)
|
| 7514 | return ret;
|
| 7515 | }
|
| 7516 | } else {
|
| 7517 |
|
| 7518 | base = nr_pages - 2 * (nr_pages - n);
|
| 7519 | l = nr_pages - n;
|
| 7520 |
|
| 7521 | for (i = 0; i < base; i++) {
|
| 7522 | cond_resched();
|
| 7523 | ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
|
| 7524 | if (ret)
|
| 7525 | return ret;
|
| 7526 | }
|
| 7527 | }
|
| 7528 | |
| 7529 | * Process remaining subpages in left-right-left-right pattern
|
| 7530 | * towards the target subpage
|
| 7531 |
|
| 7532 | for (i = 0; i < l; i++) {
|
| 7533 | int left_idx = base + i;
|
| 7534 | int right_idx = base + 2 * l - 1 - i;
|
| 7535 |
|
| 7536 | cond_resched();
|
| 7537 | ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
|
| 7538 | if (ret)
|
| 7539 | return ret;
|
| 7540 | cond_resched();
|
| 7541 | ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
|
| 7542 | if (ret)
|
| 7543 | return ret;
|
| 7544 | }
|
| 7545 | return 0;
|
| 7546 | }
|
| 7547 |
|
| 7548 | static void clear_contig_highpages(struct page *page, unsigned long addr,
|
| 7549 | unsigned int nr_pages)
|
| 7550 | {
|
| 7551 | unsigned int i, count;
|
| 7552 | |
| 7553 | * When clearing we want to operate on the largest extent possible to
|
| 7554 | * allow for architecture specific extent based optimizations.
|
| 7555 | *
|
| 7556 | * However, since clear_user_highpages() (and primitives clear_user_pages(),
|
| 7557 | * clear_pages()), do not call cond_resched(), limit the unit size when
|
| 7558 | * running under non-preemptible scheduling models.
|
| 7559 |
|
| 7560 | const unsigned int unit = preempt_model_preemptible() ?
|
| 7561 | nr_pages : PROCESS_PAGES_NON_PREEMPT_BATCH;
|
| 7562 |
|
| 7563 | might_sleep();
|
| 7564 |
|
| 7565 | for (i = 0; i < nr_pages; i += count) {
|
| 7566 | cond_resched();
|
| 7567 |
|
| 7568 | count = min(unit, nr_pages - i);
|
| 7569 | clear_user_highpages(page + i, addr + i * PAGE_SIZE, count);
|
| 7570 | }
|
| 7571 | }
|
| 7572 |
|
| 7573 | |
| 7574 | * When zeroing a folio, we want to differentiate between pages in the
|
| 7575 | * vicinity of the faulting address where we have spatial and temporal
|
| 7576 | * locality, and those far away where we don't.
|
| 7577 | *
|
| 7578 | * Use a radius of 2 for determining the local neighbourhood.
|
| 7579 |
|
| 7580 | #define FOLIO_ZERO_LOCALITY_RADIUS 2
|
| 7581 |
|
| 7582 | |
| 7583 | * folio_zero_user - Zero a folio which will be mapped to userspace.
|
| 7584 | * @folio: The folio to zero.
|
| 7585 | * @addr_hint: The address accessed by the user or the base address.
|
| 7586 |
|
| 7587 | void folio_zero_user(struct folio *folio, unsigned long addr_hint)
|
| 7588 | {
|
| 7589 | const unsigned long base_addr = ALIGN_DOWN(addr_hint, folio_size(folio));
|
| 7590 | const long fault_idx = (addr_hint - base_addr) / PAGE_SIZE;
|
| 7591 | const struct range pg = DEFINE_RANGE(0, folio_nr_pages(folio) - 1);
|
| 7592 | const long radius = FOLIO_ZERO_LOCALITY_RADIUS;
|
| 7593 | struct range r[3];
|
| 7594 | int i;
|
| 7595 |
|
| 7596 | |
| 7597 | * Faulting page and its immediate neighbourhood. Will be cleared at the
|
| 7598 | * end to keep its cachelines hot.
|
| 7599 |
|
| 7600 | r[2] = DEFINE_RANGE(fault_idx - radius < (long)pg.start ? pg.start : fault_idx - radius,
|
| 7601 | fault_idx + radius > (long)pg.end ? pg.end : fault_idx + radius);
|
| 7602 |
|
| 7603 |
|
| 7604 |
|
| 7605 | r[1] = DEFINE_RANGE(pg.start, r[2].start - 1);
|
| 7606 |
|
| 7607 |
|
| 7608 | r[0] = DEFINE_RANGE(r[2].end + 1, pg.end);
|
| 7609 |
|
| 7610 | for (i = 0; i < ARRAY_SIZE(r); i++) {
|
| 7611 | const unsigned long addr = base_addr + r[i].start * PAGE_SIZE;
|
| 7612 | const long nr_pages = (long)range_len(&r[i]);
|
| 7613 | struct page *page = folio_page(folio, r[i].start);
|
| 7614 |
|
| 7615 | if (nr_pages > 0)
|
| 7616 | clear_contig_highpages(page, addr, nr_pages);
|
| 7617 | }
|
| 7618 | }
|
| 7619 |
|
| 7620 | static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
|
| 7621 | unsigned long addr_hint,
|
| 7622 | struct vm_area_struct *vma,
|
| 7623 | unsigned int nr_pages)
|
| 7624 | {
|
| 7625 | unsigned long addr = ALIGN_DOWN(addr_hint, folio_size(dst));
|
| 7626 | struct page *dst_page;
|
| 7627 | struct page *src_page;
|
| 7628 | int i;
|
| 7629 |
|
| 7630 | for (i = 0; i < nr_pages; i++) {
|
| 7631 | dst_page = folio_page(dst, i);
|
| 7632 | src_page = folio_page(src, i);
|
| 7633 |
|
| 7634 | cond_resched();
|
| 7635 | if (copy_mc_user_highpage(dst_page, src_page,
|
| 7636 | addr + i*PAGE_SIZE, vma))
|
| 7637 | return -EHWPOISON;
|
| 7638 | }
|
| 7639 | return 0;
|
| 7640 | }
|
| 7641 |
|
| 7642 | struct copy_subpage_arg {
|
| 7643 | struct folio *dst;
|
| 7644 | struct folio *src;
|
| 7645 | struct vm_area_struct *vma;
|
| 7646 | };
|
| 7647 |
|
| 7648 | static int copy_subpage(unsigned long addr, int idx, void *arg)
|
| 7649 | {
|
| 7650 | struct copy_subpage_arg *copy_arg = arg;
|
| 7651 | struct page *dst = folio_page(copy_arg->dst, idx);
|
| 7652 | struct page *src = folio_page(copy_arg->src, idx);
|
| 7653 |
|
| 7654 | if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma))
|
| 7655 | return -EHWPOISON;
|
| 7656 | return 0;
|
| 7657 | }
|
| 7658 |
|
| 7659 | int copy_user_large_folio(struct folio *dst, struct folio *src,
|
| 7660 | unsigned long addr_hint, struct vm_area_struct *vma)
|
| 7661 | {
|
| 7662 | unsigned int nr_pages = folio_nr_pages(dst);
|
| 7663 | struct copy_subpage_arg arg = {
|
| 7664 | .dst = dst,
|
| 7665 | .src = src,
|
| 7666 | .vma = vma,
|
| 7667 | };
|
| 7668 |
|
| 7669 | if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
|
| 7670 | return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages);
|
| 7671 |
|
| 7672 | return process_huge_page(addr_hint, nr_pages, copy_subpage, &arg);
|
| 7673 | }
|
| 7674 |
|
| 7675 | long copy_folio_from_user(struct folio *dst_folio,
|
| 7676 | const void __user *usr_src,
|
| 7677 | bool allow_pagefault)
|
| 7678 | {
|
| 7679 | void *kaddr;
|
| 7680 | unsigned long i, rc = 0;
|
| 7681 | unsigned int nr_pages = folio_nr_pages(dst_folio);
|
| 7682 | unsigned long ret_val = nr_pages * PAGE_SIZE;
|
| 7683 | struct page *subpage;
|
| 7684 |
|
| 7685 | for (i = 0; i < nr_pages; i++) {
|
| 7686 | subpage = folio_page(dst_folio, i);
|
| 7687 | kaddr = kmap_local_page(subpage);
|
| 7688 | if (!allow_pagefault)
|
| 7689 | pagefault_disable();
|
| 7690 | rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
|
| 7691 | if (!allow_pagefault)
|
| 7692 | pagefault_enable();
|
| 7693 | kunmap_local(kaddr);
|
| 7694 |
|
| 7695 | ret_val -= (PAGE_SIZE - rc);
|
| 7696 | if (rc)
|
| 7697 | break;
|
| 7698 |
|
| 7699 | flush_dcache_page(subpage);
|
| 7700 |
|
| 7701 | cond_resched();
|
| 7702 | }
|
| 7703 | return ret_val;
|
| 7704 | }
|
| 7705 | #endif
|
| 7706 |
|
| 7707 | #if defined(CONFIG_SPLIT_PTE_PTLOCKS) && ALLOC_SPLIT_PTLOCKS
|
| 7708 |
|
| 7709 | static struct kmem_cache *page_ptl_cachep;
|
| 7710 |
|
| 7711 | void __init ptlock_cache_init(void)
|
| 7712 | {
|
| 7713 | page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
|
| 7714 | SLAB_PANIC, NULL);
|
| 7715 | }
|
| 7716 |
|
| 7717 | bool ptlock_alloc(struct ptdesc *ptdesc)
|
| 7718 | {
|
| 7719 | spinlock_t *ptl;
|
| 7720 |
|
| 7721 | ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
|
| 7722 | if (!ptl)
|
| 7723 | return false;
|
| 7724 | ptdesc->ptl = ptl;
|
| 7725 | return true;
|
| 7726 | }
|
| 7727 |
|
| 7728 | void ptlock_free(struct ptdesc *ptdesc)
|
| 7729 | {
|
| 7730 | if (ptdesc->ptl)
|
| 7731 | kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
|
| 7732 | }
|
| 7733 | #endif
|
| 7734 |
|
| 7735 | void vma_pgtable_walk_begin(struct vm_area_struct *vma)
|
| 7736 | {
|
| 7737 | if (is_vm_hugetlb_page(vma))
|
| 7738 | hugetlb_vma_lock_read(vma);
|
| 7739 | }
|
| 7740 |
|
| 7741 | void vma_pgtable_walk_end(struct vm_area_struct *vma)
|
| 7742 | {
|
| 7743 | if (is_vm_hugetlb_page(vma))
|
| 7744 | hugetlb_vma_unlock_read(vma);
|
| 7745 | }
|