| 1 | |
| 2 | |
| 3 | * mm/userfaultfd.c
|
| 4 | *
|
| 5 | * Copyright (C) 2007 Davide Libenzi <[email protected]>
|
| 6 | * Copyright (C) 2008-2009 Red Hat, Inc.
|
| 7 | * Copyright (C) 2015 Red Hat, Inc.
|
| 8 | *
|
| 9 | * Some part derived from fs/eventfd.c (anon inode setup) and
|
| 10 | * mm/ksm.c (mm hashing).
|
| 11 |
|
| 12 |
|
| 13 | #include <linux/mm.h>
|
| 14 | #include <linux/sched/signal.h>
|
| 15 | #include <linux/pagemap.h>
|
| 16 | #include <linux/rmap.h>
|
| 17 | #include <linux/swap.h>
|
| 18 | #include <linux/leafops.h>
|
| 19 | #include <linux/userfaultfd_k.h>
|
| 20 | #include <linux/mmu_notifier.h>
|
| 21 | #include <linux/hugetlb.h>
|
| 22 | #include <linux/list.h>
|
| 23 | #include <linux/sched/mm.h>
|
| 24 | #include <linux/mm_inline.h>
|
| 25 | #include <linux/poll.h>
|
| 26 | #include <linux/slab.h>
|
| 27 | #include <linux/seq_file.h>
|
| 28 | #include <linux/bug.h>
|
| 29 | #include <linux/anon_inodes.h>
|
| 30 | #include <linux/syscalls.h>
|
| 31 | #include <linux/miscdevice.h>
|
| 32 | #include <linux/uio.h>
|
| 33 | #include <linux/file.h>
|
| 34 | #include <linux/cleanup.h>
|
| 35 | #include <asm/tlbflush.h>
|
| 36 | #include <asm/tlb.h>
|
| 37 | #include "internal.h"
|
| 38 | #include "swap.h"
|
| 39 |
|
| 40 | struct mfill_state {
|
| 41 | struct userfaultfd_ctx *ctx;
|
| 42 | unsigned long src_start;
|
| 43 | unsigned long dst_start;
|
| 44 | unsigned long len;
|
| 45 | uffd_flags_t flags;
|
| 46 |
|
| 47 | struct vm_area_struct *vma;
|
| 48 | unsigned long src_addr;
|
| 49 | unsigned long dst_addr;
|
| 50 | pmd_t *pmd;
|
| 51 | };
|
| 52 |
|
| 53 | static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags)
|
| 54 | {
|
| 55 |
|
| 56 | if (vm_flags & VM_UFFD_MINOR)
|
| 57 | return false;
|
| 58 | return true;
|
| 59 | }
|
| 60 |
|
| 61 | static struct folio *anon_alloc_folio(struct vm_area_struct *vma,
|
| 62 | unsigned long addr)
|
| 63 | {
|
| 64 | struct folio *folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma,
|
| 65 | addr);
|
| 66 |
|
| 67 | if (!folio)
|
| 68 | return NULL;
|
| 69 |
|
| 70 | if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL)) {
|
| 71 | folio_put(folio);
|
| 72 | return NULL;
|
| 73 | }
|
| 74 |
|
| 75 | return folio;
|
| 76 | }
|
| 77 |
|
| 78 | static const struct vm_uffd_ops anon_uffd_ops = {
|
| 79 | .can_userfault = anon_can_userfault,
|
| 80 | .alloc_folio = anon_alloc_folio,
|
| 81 | };
|
| 82 |
|
| 83 | static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)
|
| 84 | {
|
| 85 | if (vma_is_anonymous(vma))
|
| 86 | return &anon_uffd_ops;
|
| 87 | return vma->vm_ops->uffd_ops;
|
| 88 | }
|
| 89 |
|
| 90 | static __always_inline
|
| 91 | bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end)
|
| 92 | {
|
| 93 |
|
| 94 | if (dst_end > dst_vma->vm_end)
|
| 95 | return false;
|
| 96 |
|
| 97 | |
| 98 | * Check the vma is registered in uffd, this is required to
|
| 99 | * enforce the VM_MAYWRITE check done at uffd registration
|
| 100 | * time.
|
| 101 |
|
| 102 | if (!dst_vma->vm_userfaultfd_ctx.ctx)
|
| 103 | return false;
|
| 104 |
|
| 105 | return true;
|
| 106 | }
|
| 107 |
|
| 108 | static __always_inline
|
| 109 | struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm,
|
| 110 | unsigned long addr)
|
| 111 | {
|
| 112 | struct vm_area_struct *vma;
|
| 113 |
|
| 114 | mmap_assert_locked(mm);
|
| 115 | vma = vma_lookup(mm, addr);
|
| 116 | if (!vma)
|
| 117 | vma = ERR_PTR(-ENOENT);
|
| 118 | else if (!(vma->vm_flags & VM_SHARED) &&
|
| 119 | unlikely(anon_vma_prepare(vma)))
|
| 120 | vma = ERR_PTR(-ENOMEM);
|
| 121 |
|
| 122 | return vma;
|
| 123 | }
|
| 124 |
|
| 125 | #ifdef CONFIG_PER_VMA_LOCK
|
| 126 | |
| 127 | * uffd_lock_vma() - Lookup and lock vma corresponding to @address.
|
| 128 | * @mm: mm to search vma in.
|
| 129 | * @address: address that the vma should contain.
|
| 130 | *
|
| 131 | * Should be called without holding mmap_lock.
|
| 132 | *
|
| 133 | * Return: A locked vma containing @address, -ENOENT if no vma is found, or
|
| 134 | * -ENOMEM if anon_vma couldn't be allocated.
|
| 135 |
|
| 136 | static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm,
|
| 137 | unsigned long address)
|
| 138 | {
|
| 139 | struct vm_area_struct *vma;
|
| 140 |
|
| 141 | vma = lock_vma_under_rcu(mm, address);
|
| 142 | if (vma) {
|
| 143 | |
| 144 | * We know we're going to need to use anon_vma, so check
|
| 145 | * that early.
|
| 146 |
|
| 147 | if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma))
|
| 148 | vma_end_read(vma);
|
| 149 | else
|
| 150 | return vma;
|
| 151 | }
|
| 152 |
|
| 153 | mmap_read_lock(mm);
|
| 154 | vma = find_vma_and_prepare_anon(mm, address);
|
| 155 | if (!IS_ERR(vma)) {
|
| 156 | bool locked = vma_start_read_locked(vma);
|
| 157 |
|
| 158 | if (!locked)
|
| 159 | vma = ERR_PTR(-EAGAIN);
|
| 160 | }
|
| 161 |
|
| 162 | mmap_read_unlock(mm);
|
| 163 | return vma;
|
| 164 | }
|
| 165 |
|
| 166 | static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm,
|
| 167 | unsigned long dst_start,
|
| 168 | unsigned long len)
|
| 169 | {
|
| 170 | struct vm_area_struct *dst_vma;
|
| 171 |
|
| 172 | dst_vma = uffd_lock_vma(dst_mm, dst_start);
|
| 173 | if (IS_ERR(dst_vma) || validate_dst_vma(dst_vma, dst_start + len))
|
| 174 | return dst_vma;
|
| 175 |
|
| 176 | vma_end_read(dst_vma);
|
| 177 | return ERR_PTR(-ENOENT);
|
| 178 | }
|
| 179 |
|
| 180 | static void uffd_mfill_unlock(struct vm_area_struct *vma)
|
| 181 | {
|
| 182 | vma_end_read(vma);
|
| 183 | }
|
| 184 |
|
| 185 | #else
|
| 186 |
|
| 187 | static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm,
|
| 188 | unsigned long dst_start,
|
| 189 | unsigned long len)
|
| 190 | {
|
| 191 | struct vm_area_struct *dst_vma;
|
| 192 |
|
| 193 | mmap_read_lock(dst_mm);
|
| 194 | dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start);
|
| 195 | if (IS_ERR(dst_vma))
|
| 196 | goto out_unlock;
|
| 197 |
|
| 198 | if (validate_dst_vma(dst_vma, dst_start + len))
|
| 199 | return dst_vma;
|
| 200 |
|
| 201 | dst_vma = ERR_PTR(-ENOENT);
|
| 202 | out_unlock:
|
| 203 | mmap_read_unlock(dst_mm);
|
| 204 | return dst_vma;
|
| 205 | }
|
| 206 |
|
| 207 | static void uffd_mfill_unlock(struct vm_area_struct *vma)
|
| 208 | {
|
| 209 | mmap_read_unlock(vma->vm_mm);
|
| 210 | }
|
| 211 | #endif
|
| 212 |
|
| 213 | static void mfill_put_vma(struct mfill_state *state)
|
| 214 | {
|
| 215 | if (!state->vma)
|
| 216 | return;
|
| 217 |
|
| 218 | up_read(&state->ctx->map_changing_lock);
|
| 219 | uffd_mfill_unlock(state->vma);
|
| 220 | state->vma = NULL;
|
| 221 | }
|
| 222 |
|
| 223 | static int mfill_get_vma(struct mfill_state *state)
|
| 224 | {
|
| 225 | struct userfaultfd_ctx *ctx = state->ctx;
|
| 226 | uffd_flags_t flags = state->flags;
|
| 227 | struct vm_area_struct *dst_vma;
|
| 228 | const struct vm_uffd_ops *ops;
|
| 229 | int err;
|
| 230 |
|
| 231 | |
| 232 | * Make sure the vma is not shared, that the dst range is
|
| 233 | * both valid and fully within a single existing vma.
|
| 234 |
|
| 235 | dst_vma = uffd_mfill_lock(ctx->mm, state->dst_start, state->len);
|
| 236 | if (IS_ERR(dst_vma))
|
| 237 | return PTR_ERR(dst_vma);
|
| 238 |
|
| 239 | |
| 240 | * If memory mappings are changing because of non-cooperative
|
| 241 | * operation (e.g. mremap) running in parallel, bail out and
|
| 242 | * request the user to retry later
|
| 243 |
|
| 244 | down_read(&ctx->map_changing_lock);
|
| 245 | state->vma = dst_vma;
|
| 246 | err = -EAGAIN;
|
| 247 | if (atomic_read(&ctx->mmap_changing))
|
| 248 | goto out_unlock;
|
| 249 |
|
| 250 | err = -EINVAL;
|
| 251 |
|
| 252 | |
| 253 | * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
|
| 254 | * it will overwrite vm_ops, so vma_is_anonymous must return false.
|
| 255 |
|
| 256 | if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) &&
|
| 257 | dst_vma->vm_flags & VM_SHARED))
|
| 258 | goto out_unlock;
|
| 259 |
|
| 260 | |
| 261 | * validate 'mode' now that we know the dst_vma: don't allow
|
| 262 | * a wrprotect copy if the userfaultfd didn't register as WP.
|
| 263 |
|
| 264 | if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP))
|
| 265 | goto out_unlock;
|
| 266 |
|
| 267 | if (is_vm_hugetlb_page(dst_vma))
|
| 268 | return 0;
|
| 269 |
|
| 270 | ops = vma_uffd_ops(dst_vma);
|
| 271 | if (!ops)
|
| 272 | goto out_unlock;
|
| 273 |
|
| 274 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE) &&
|
| 275 | !ops->get_folio_noalloc)
|
| 276 | goto out_unlock;
|
| 277 |
|
| 278 | return 0;
|
| 279 |
|
| 280 | out_unlock:
|
| 281 | mfill_put_vma(state);
|
| 282 | return err;
|
| 283 | }
|
| 284 |
|
| 285 | static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
|
| 286 | {
|
| 287 | pgd_t *pgd;
|
| 288 | p4d_t *p4d;
|
| 289 | pud_t *pud;
|
| 290 |
|
| 291 | pgd = pgd_offset(mm, address);
|
| 292 | p4d = p4d_alloc(mm, pgd, address);
|
| 293 | if (!p4d)
|
| 294 | return NULL;
|
| 295 | pud = pud_alloc(mm, p4d, address);
|
| 296 | if (!pud)
|
| 297 | return NULL;
|
| 298 | |
| 299 | * Note that we didn't run this because the pmd was
|
| 300 | * missing, the *pmd may be already established and in
|
| 301 | * turn it may also be a trans_huge_pmd.
|
| 302 |
|
| 303 | return pmd_alloc(mm, pud, address);
|
| 304 | }
|
| 305 |
|
| 306 | static int mfill_establish_pmd(struct mfill_state *state)
|
| 307 | {
|
| 308 | struct mm_struct *dst_mm = state->ctx->mm;
|
| 309 | pmd_t *dst_pmd, dst_pmdval;
|
| 310 |
|
| 311 | dst_pmd = mm_alloc_pmd(dst_mm, state->dst_addr);
|
| 312 | if (unlikely(!dst_pmd))
|
| 313 | return -ENOMEM;
|
| 314 |
|
| 315 | dst_pmdval = pmdp_get_lockless(dst_pmd);
|
| 316 | if (unlikely(pmd_none(dst_pmdval)) &&
|
| 317 | unlikely(__pte_alloc(dst_mm, dst_pmd)))
|
| 318 | return -ENOMEM;
|
| 319 |
|
| 320 | dst_pmdval = pmdp_get_lockless(dst_pmd);
|
| 321 | |
| 322 | * If the dst_pmd is THP don't override it and just be strict.
|
| 323 | * (This includes the case where the PMD used to be THP and
|
| 324 | * changed back to none after __pte_alloc().)
|
| 325 |
|
| 326 | if (unlikely(!pmd_present(dst_pmdval) || pmd_leaf(dst_pmdval)))
|
| 327 | return -EEXIST;
|
| 328 | if (unlikely(pmd_bad(dst_pmdval)))
|
| 329 | return -EFAULT;
|
| 330 |
|
| 331 | state->pmd = dst_pmd;
|
| 332 | return 0;
|
| 333 | }
|
| 334 |
|
| 335 |
|
| 336 | static bool mfill_file_over_size(struct vm_area_struct *dst_vma,
|
| 337 | unsigned long dst_addr)
|
| 338 | {
|
| 339 | struct inode *inode;
|
| 340 | pgoff_t offset, max_off;
|
| 341 |
|
| 342 | if (!dst_vma->vm_file)
|
| 343 | return false;
|
| 344 |
|
| 345 | inode = dst_vma->vm_file->f_inode;
|
| 346 | offset = linear_page_index(dst_vma, dst_addr);
|
| 347 | max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
|
| 348 | return offset >= max_off;
|
| 349 | }
|
| 350 |
|
| 351 | |
| 352 | * Install PTEs, to map dst_addr (within dst_vma) to page.
|
| 353 | *
|
| 354 | * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem
|
| 355 | * and anon, and for both shared and private VMAs.
|
| 356 |
|
| 357 | static int mfill_atomic_install_pte(pmd_t *dst_pmd,
|
| 358 | struct vm_area_struct *dst_vma,
|
| 359 | unsigned long dst_addr, struct page *page,
|
| 360 | uffd_flags_t flags)
|
| 361 | {
|
| 362 | int ret;
|
| 363 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 364 | pte_t _dst_pte, *dst_pte;
|
| 365 | bool writable = dst_vma->vm_flags & VM_WRITE;
|
| 366 | bool vm_shared = dst_vma->vm_flags & VM_SHARED;
|
| 367 | spinlock_t *ptl;
|
| 368 | struct folio *folio = page_folio(page);
|
| 369 | bool page_in_cache = folio_mapping(folio);
|
| 370 | pte_t dst_ptep;
|
| 371 |
|
| 372 | _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
|
| 373 | _dst_pte = pte_mkdirty(_dst_pte);
|
| 374 | if (page_in_cache && !vm_shared)
|
| 375 | writable = false;
|
| 376 | if (writable)
|
| 377 | _dst_pte = pte_mkwrite(_dst_pte, dst_vma);
|
| 378 | if (flags & MFILL_ATOMIC_WP)
|
| 379 | _dst_pte = pte_mkuffd(_dst_pte);
|
| 380 |
|
| 381 | ret = -EAGAIN;
|
| 382 | dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
|
| 383 | if (!dst_pte)
|
| 384 | goto out;
|
| 385 |
|
| 386 | if (mfill_file_over_size(dst_vma, dst_addr)) {
|
| 387 | ret = -EFAULT;
|
| 388 | goto out_unlock;
|
| 389 | }
|
| 390 |
|
| 391 | ret = -EEXIST;
|
| 392 |
|
| 393 | dst_ptep = ptep_get(dst_pte);
|
| 394 |
|
| 395 | |
| 396 | * We are allowed to overwrite a UFFD pte marker: consider when both
|
| 397 | * MISSING|WP registered, we firstly wr-protect a none pte which has no
|
| 398 | * page cache page backing it, then access the page.
|
| 399 |
|
| 400 | if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep))
|
| 401 | goto out_unlock;
|
| 402 |
|
| 403 | if (page_in_cache) {
|
| 404 | folio_add_file_rmap_pte(folio, page, dst_vma);
|
| 405 | } else {
|
| 406 | folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE);
|
| 407 | folio_add_lru_vma(folio, dst_vma);
|
| 408 | }
|
| 409 |
|
| 410 | |
| 411 | * Must happen after rmap, as mm_counter() checks mapping (via
|
| 412 | * PageAnon()), which is set by __page_set_anon_rmap().
|
| 413 |
|
| 414 | inc_mm_counter(dst_mm, mm_counter(folio));
|
| 415 |
|
| 416 | set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
|
| 417 |
|
| 418 | if (page_in_cache)
|
| 419 | folio_unlock(folio);
|
| 420 |
|
| 421 |
|
| 422 | update_mmu_cache(dst_vma, dst_addr, dst_pte);
|
| 423 | ret = 0;
|
| 424 | out_unlock:
|
| 425 | pte_unmap_unlock(dst_pte, ptl);
|
| 426 | out:
|
| 427 | return ret;
|
| 428 | }
|
| 429 |
|
| 430 | static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
|
| 431 | {
|
| 432 | void *kaddr;
|
| 433 | int ret;
|
| 434 |
|
| 435 | kaddr = kmap_local_folio(folio, 0);
|
| 436 | |
| 437 | * The read mmap_lock is held here. Despite the
|
| 438 | * mmap_lock being read recursive a deadlock is still
|
| 439 | * possible if a writer has taken a lock. For example:
|
| 440 | *
|
| 441 | * process A thread 1 takes read lock on own mmap_lock
|
| 442 | * process A thread 2 calls mmap, blocks taking write lock
|
| 443 | * process B thread 1 takes page fault, read lock on own mmap lock
|
| 444 | * process B thread 2 calls mmap, blocks taking write lock
|
| 445 | * process A thread 1 blocks taking read lock on process B
|
| 446 | * process B thread 1 blocks taking read lock on process A
|
| 447 | *
|
| 448 | * Disable page faults to prevent potential deadlock
|
| 449 | * and retry the copy outside the mmap_lock.
|
| 450 |
|
| 451 | pagefault_disable();
|
| 452 | ret = copy_from_user(kaddr, (const void __user *) src_addr,
|
| 453 | PAGE_SIZE);
|
| 454 | pagefault_enable();
|
| 455 | kunmap_local(kaddr);
|
| 456 |
|
| 457 | if (ret)
|
| 458 | return -EFAULT;
|
| 459 |
|
| 460 | flush_dcache_folio(folio);
|
| 461 | return ret;
|
| 462 | }
|
| 463 |
|
| 464 | #define MFILL_RETRY_STATE_VMA_FLAGS \
|
| 465 | append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT)
|
| 466 |
|
| 467 | |
| 468 | * VMA state saved before dropping the locks in mfill_copy_folio_retry().
|
| 469 | * Used to detect VMA replacement or incompatible changes after reacquiring the
|
| 470 | * locks.
|
| 471 |
|
| 472 | struct mfill_retry_state {
|
| 473 | const struct vm_uffd_ops *ops;
|
| 474 | struct file *file;
|
| 475 | vma_flags_t flags;
|
| 476 | pgoff_t pgoff;
|
| 477 | };
|
| 478 |
|
| 479 | static void mfill_retry_state_save(struct mfill_retry_state *s,
|
| 480 | struct vm_area_struct *vma)
|
| 481 | {
|
| 482 | s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS);
|
| 483 | s->ops = vma_uffd_ops(vma);
|
| 484 | s->pgoff = vma_start_pgoff(vma);
|
| 485 |
|
| 486 | if (vma->vm_file)
|
| 487 | s->file = get_file(vma->vm_file);
|
| 488 | }
|
| 489 |
|
| 490 | static bool mfill_retry_state_changed(struct mfill_retry_state *state,
|
| 491 | struct vm_area_struct *vma)
|
| 492 | {
|
| 493 | vma_flags_t flags = vma_flags_and_mask(&vma->flags,
|
| 494 | MFILL_RETRY_STATE_VMA_FLAGS);
|
| 495 |
|
| 496 |
|
| 497 | if (!vma_flags_same_pair(&state->flags, &flags))
|
| 498 | return true;
|
| 499 |
|
| 500 |
|
| 501 | if (state->ops != vma_uffd_ops(vma))
|
| 502 | return true;
|
| 503 |
|
| 504 |
|
| 505 | if (!state->file)
|
| 506 | return !vma_is_anonymous(vma);
|
| 507 |
|
| 508 |
|
| 509 | if (!vma->vm_file || vma->vm_file->f_inode != state->file->f_inode ||
|
| 510 | state->file != vma->vm_file || vma_start_pgoff(vma) != state->pgoff)
|
| 511 | return true;
|
| 512 |
|
| 513 | return false;
|
| 514 | }
|
| 515 |
|
| 516 | static void mfill_retry_state_put(struct mfill_retry_state *s)
|
| 517 | {
|
| 518 | if (s->file)
|
| 519 | fput(s->file);
|
| 520 | }
|
| 521 |
|
| 522 | DEFINE_FREE(retry_put, struct mfill_retry_state *,
|
| 523 | if (_T) mfill_retry_state_put(_T));
|
| 524 |
|
| 525 | static int mfill_copy_folio_retry(struct mfill_state *mfill_state,
|
| 526 | struct folio *folio)
|
| 527 | {
|
| 528 | struct mfill_retry_state retry_state = { 0 };
|
| 529 | struct mfill_retry_state *for_free __free(retry_put) = &retry_state;
|
| 530 | unsigned long src_addr = mfill_state->src_addr;
|
| 531 | void *kaddr;
|
| 532 | int err;
|
| 533 |
|
| 534 | mfill_retry_state_save(&retry_state, mfill_state->vma);
|
| 535 |
|
| 536 |
|
| 537 | mfill_put_vma(mfill_state);
|
| 538 |
|
| 539 | kaddr = kmap_local_folio(folio, 0);
|
| 540 | err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE);
|
| 541 | kunmap_local(kaddr);
|
| 542 | if (unlikely(err))
|
| 543 | return -EFAULT;
|
| 544 |
|
| 545 | flush_dcache_folio(folio);
|
| 546 |
|
| 547 |
|
| 548 | err = mfill_get_vma(mfill_state);
|
| 549 | if (err)
|
| 550 | return err;
|
| 551 |
|
| 552 | if (mfill_retry_state_changed(&retry_state, mfill_state->vma))
|
| 553 | return -EAGAIN;
|
| 554 |
|
| 555 | err = mfill_establish_pmd(mfill_state);
|
| 556 | if (err)
|
| 557 | return err;
|
| 558 |
|
| 559 | return 0;
|
| 560 | }
|
| 561 |
|
| 562 | static int __mfill_atomic_pte(struct mfill_state *state,
|
| 563 | const struct vm_uffd_ops *ops)
|
| 564 | {
|
| 565 | unsigned long dst_addr = state->dst_addr;
|
| 566 | unsigned long src_addr = state->src_addr;
|
| 567 | uffd_flags_t flags = state->flags;
|
| 568 | struct folio *folio;
|
| 569 | int ret;
|
| 570 |
|
| 571 | if (!ops) {
|
| 572 | VM_WARN_ONCE(1, "UFFDIO_COPY for unsupported VMA");
|
| 573 | return -EOPNOTSUPP;
|
| 574 | }
|
| 575 |
|
| 576 | folio = ops->alloc_folio(state->vma, state->dst_addr);
|
| 577 | if (!folio)
|
| 578 | return -ENOMEM;
|
| 579 |
|
| 580 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) {
|
| 581 | ret = mfill_copy_folio_locked(folio, src_addr);
|
| 582 | |
| 583 | * Fallback to copy_from_user outside mmap_lock.
|
| 584 | * If retry is successful, mfill_copy_folio_locked() returns
|
| 585 | * with locks retaken by mfill_get_vma().
|
| 586 | * If there was an error, we must mfill_put_vma() anyway and it
|
| 587 | * will take care of unlocking if needed.
|
| 588 |
|
| 589 | if (unlikely(ret)) {
|
| 590 | ret = mfill_copy_folio_retry(state, folio);
|
| 591 | if (ret)
|
| 592 | goto err_folio_put;
|
| 593 | }
|
| 594 | } else if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE)) {
|
| 595 | clear_user_highpage(&folio->page, state->dst_addr);
|
| 596 | } else {
|
| 597 | VM_WARN_ONCE(1, "Unknown UFFDIO operation, flags: %x", flags);
|
| 598 | }
|
| 599 |
|
| 600 | |
| 601 | * The memory barrier inside __folio_mark_uptodate makes sure that
|
| 602 | * preceding stores to the page contents become visible before
|
| 603 | * the set_pte_at() write.
|
| 604 |
|
| 605 | __folio_mark_uptodate(folio);
|
| 606 |
|
| 607 | if (ops->filemap_add) {
|
| 608 | ret = ops->filemap_add(folio, state->vma, state->dst_addr);
|
| 609 | if (ret)
|
| 610 | goto err_folio_put;
|
| 611 | }
|
| 612 |
|
| 613 | ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr,
|
| 614 | &folio->page, flags);
|
| 615 | if (ret)
|
| 616 | goto err_filemap_remove;
|
| 617 |
|
| 618 | return 0;
|
| 619 |
|
| 620 | err_filemap_remove:
|
| 621 | if (ops->filemap_remove)
|
| 622 | ops->filemap_remove(folio, state->vma);
|
| 623 | err_folio_put:
|
| 624 | folio_put(folio);
|
| 625 | return ret;
|
| 626 | }
|
| 627 |
|
| 628 | static int mfill_atomic_pte_copy(struct mfill_state *state)
|
| 629 | {
|
| 630 | const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma);
|
| 631 |
|
| 632 | |
| 633 | * The normal page fault path for a MAP_PRIVATE mapping in a
|
| 634 | * file-backed VMA will invoke the fault, fill the hole in the file and
|
| 635 | * COW it right away. The result generates plain anonymous memory.
|
| 636 | * So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll
|
| 637 | * generate anonymous memory directly without actually filling the
|
| 638 | * hole. For the MAP_PRIVATE case the robustness check only happens in
|
| 639 | * the pagetable (to verify it's still none) and not in the page cache.
|
| 640 |
|
| 641 | if (!(state->vma->vm_flags & VM_SHARED))
|
| 642 | ops = &anon_uffd_ops;
|
| 643 |
|
| 644 | return __mfill_atomic_pte(state, ops);
|
| 645 | }
|
| 646 |
|
| 647 | static int mfill_atomic_pte_zeroed_folio(struct mfill_state *state)
|
| 648 | {
|
| 649 | const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma);
|
| 650 |
|
| 651 | return __mfill_atomic_pte(state, ops);
|
| 652 | }
|
| 653 |
|
| 654 | static int mfill_atomic_pte_zeropage(struct mfill_state *state)
|
| 655 | {
|
| 656 | struct vm_area_struct *dst_vma = state->vma;
|
| 657 | unsigned long dst_addr = state->dst_addr;
|
| 658 | pmd_t *dst_pmd = state->pmd;
|
| 659 | pte_t _dst_pte, *dst_pte;
|
| 660 | spinlock_t *ptl;
|
| 661 | int ret;
|
| 662 |
|
| 663 | if (mm_forbids_zeropage(dst_vma->vm_mm) ||
|
| 664 | (dst_vma->vm_flags & VM_SHARED))
|
| 665 | return mfill_atomic_pte_zeroed_folio(state);
|
| 666 |
|
| 667 | _dst_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr),
|
| 668 | dst_vma->vm_page_prot));
|
| 669 | ret = -EAGAIN;
|
| 670 | dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl);
|
| 671 | if (!dst_pte)
|
| 672 | goto out;
|
| 673 | if (mfill_file_over_size(dst_vma, dst_addr)) {
|
| 674 | ret = -EFAULT;
|
| 675 | goto out_unlock;
|
| 676 | }
|
| 677 | ret = -EEXIST;
|
| 678 | if (!pte_none(ptep_get(dst_pte)))
|
| 679 | goto out_unlock;
|
| 680 | set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte);
|
| 681 |
|
| 682 | update_mmu_cache(dst_vma, dst_addr, dst_pte);
|
| 683 | ret = 0;
|
| 684 | out_unlock:
|
| 685 | pte_unmap_unlock(dst_pte, ptl);
|
| 686 | out:
|
| 687 | return ret;
|
| 688 | }
|
| 689 |
|
| 690 |
|
| 691 | static int mfill_atomic_pte_continue(struct mfill_state *state)
|
| 692 | {
|
| 693 | struct vm_area_struct *dst_vma = state->vma;
|
| 694 | const struct vm_uffd_ops *ops = vma_uffd_ops(dst_vma);
|
| 695 | unsigned long dst_addr = state->dst_addr;
|
| 696 | pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
|
| 697 | struct inode *inode = file_inode(dst_vma->vm_file);
|
| 698 | uffd_flags_t flags = state->flags;
|
| 699 | pmd_t *dst_pmd = state->pmd;
|
| 700 | struct folio *folio;
|
| 701 | struct page *page;
|
| 702 | int ret;
|
| 703 |
|
| 704 | if (!ops) {
|
| 705 | VM_WARN_ONCE(1, "UFFDIO_CONTINUE for unsupported VMA");
|
| 706 | return -EOPNOTSUPP;
|
| 707 | }
|
| 708 |
|
| 709 | folio = ops->get_folio_noalloc(inode, pgoff);
|
| 710 |
|
| 711 | if (IS_ERR_OR_NULL(folio))
|
| 712 | return -EFAULT;
|
| 713 |
|
| 714 | page = folio_file_page(folio, pgoff);
|
| 715 | if (PageHWPoison(page)) {
|
| 716 | ret = -EIO;
|
| 717 | goto out_release;
|
| 718 | }
|
| 719 |
|
| 720 | ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,
|
| 721 | page, flags);
|
| 722 | if (ret)
|
| 723 | goto out_release;
|
| 724 |
|
| 725 | return 0;
|
| 726 |
|
| 727 | out_release:
|
| 728 | folio_unlock(folio);
|
| 729 | folio_put(folio);
|
| 730 | return ret;
|
| 731 | }
|
| 732 |
|
| 733 |
|
| 734 | static int mfill_atomic_pte_poison(struct mfill_state *state)
|
| 735 | {
|
| 736 | struct vm_area_struct *dst_vma = state->vma;
|
| 737 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 738 | unsigned long dst_addr = state->dst_addr;
|
| 739 | pmd_t *dst_pmd = state->pmd;
|
| 740 | pte_t _dst_pte, *dst_pte;
|
| 741 | spinlock_t *ptl;
|
| 742 | int ret;
|
| 743 |
|
| 744 | _dst_pte = make_pte_marker(PTE_MARKER_POISONED);
|
| 745 | ret = -EAGAIN;
|
| 746 | dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
|
| 747 | if (!dst_pte)
|
| 748 | goto out;
|
| 749 |
|
| 750 | if (mfill_file_over_size(dst_vma, dst_addr)) {
|
| 751 | ret = -EFAULT;
|
| 752 | goto out_unlock;
|
| 753 | }
|
| 754 |
|
| 755 | ret = -EEXIST;
|
| 756 |
|
| 757 | if (!pte_none(ptep_get(dst_pte)))
|
| 758 | goto out_unlock;
|
| 759 |
|
| 760 | set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
|
| 761 |
|
| 762 |
|
| 763 | update_mmu_cache(dst_vma, dst_addr, dst_pte);
|
| 764 | ret = 0;
|
| 765 | out_unlock:
|
| 766 | pte_unmap_unlock(dst_pte, ptl);
|
| 767 | out:
|
| 768 | return ret;
|
| 769 | }
|
| 770 |
|
| 771 | #ifdef CONFIG_HUGETLB_PAGE
|
| 772 | |
| 773 | * mfill_atomic processing for HUGETLB vmas. Note that this routine is
|
| 774 | * called with either vma-lock or mmap_lock held, it will release the lock
|
| 775 | * before returning.
|
| 776 |
|
| 777 | static __always_inline ssize_t mfill_atomic_hugetlb(
|
| 778 | struct userfaultfd_ctx *ctx,
|
| 779 | struct vm_area_struct *dst_vma,
|
| 780 | unsigned long dst_start,
|
| 781 | unsigned long src_start,
|
| 782 | unsigned long len,
|
| 783 | uffd_flags_t flags)
|
| 784 | {
|
| 785 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 786 | ssize_t err;
|
| 787 | pte_t *dst_pte;
|
| 788 | unsigned long src_addr, dst_addr;
|
| 789 | long copied;
|
| 790 | struct folio *folio;
|
| 791 | unsigned long vma_hpagesize;
|
| 792 | pgoff_t idx;
|
| 793 | u32 hash;
|
| 794 | struct address_space *mapping;
|
| 795 |
|
| 796 | |
| 797 | * There is no default zero huge page for all huge page sizes as
|
| 798 | * supported by hugetlb. A PMD_SIZE huge pages may exist as used
|
| 799 | * by THP. Since we can not reliably insert a zero page, this
|
| 800 | * feature is not supported.
|
| 801 |
|
| 802 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE)) {
|
| 803 | up_read(&ctx->map_changing_lock);
|
| 804 | uffd_mfill_unlock(dst_vma);
|
| 805 | return -EINVAL;
|
| 806 | }
|
| 807 |
|
| 808 | src_addr = src_start;
|
| 809 | dst_addr = dst_start;
|
| 810 | copied = 0;
|
| 811 | folio = NULL;
|
| 812 | vma_hpagesize = vma_kernel_pagesize(dst_vma);
|
| 813 |
|
| 814 | |
| 815 | * Validate alignment based on huge page size
|
| 816 |
|
| 817 | err = -EINVAL;
|
| 818 | if (dst_start & (vma_hpagesize - 1) || len & (vma_hpagesize - 1))
|
| 819 | goto out_unlock;
|
| 820 |
|
| 821 | retry:
|
| 822 | |
| 823 | * On routine entry dst_vma is set. If we had to drop mmap_lock and
|
| 824 | * retry, dst_vma will be set to NULL and we must lookup again.
|
| 825 |
|
| 826 | if (!dst_vma) {
|
| 827 | dst_vma = uffd_mfill_lock(dst_mm, dst_start, len);
|
| 828 | if (IS_ERR(dst_vma)) {
|
| 829 | err = PTR_ERR(dst_vma);
|
| 830 | goto out;
|
| 831 | }
|
| 832 |
|
| 833 | err = -ENOENT;
|
| 834 | if (!is_vm_hugetlb_page(dst_vma))
|
| 835 | goto out_unlock_vma;
|
| 836 |
|
| 837 | err = -EINVAL;
|
| 838 | if (vma_hpagesize != vma_kernel_pagesize(dst_vma))
|
| 839 | goto out_unlock_vma;
|
| 840 |
|
| 841 | |
| 842 | * If memory mappings are changing because of non-cooperative
|
| 843 | * operation (e.g. mremap) running in parallel, bail out and
|
| 844 | * request the user to retry later
|
| 845 |
|
| 846 | down_read(&ctx->map_changing_lock);
|
| 847 | err = -EAGAIN;
|
| 848 | if (atomic_read(&ctx->mmap_changing))
|
| 849 | goto out_unlock;
|
| 850 | }
|
| 851 |
|
| 852 | while (src_addr < src_start + len) {
|
| 853 | VM_WARN_ON_ONCE(dst_addr >= dst_start + len);
|
| 854 |
|
| 855 | |
| 856 | * Serialize via vma_lock and hugetlb_fault_mutex.
|
| 857 | * vma_lock ensures the dst_pte remains valid even
|
| 858 | * in the case of shared pmds. fault mutex prevents
|
| 859 | * races with other faulting threads.
|
| 860 |
|
| 861 | idx = hugetlb_linear_page_index(dst_vma, dst_addr);
|
| 862 | mapping = dst_vma->vm_file->f_mapping;
|
| 863 | hash = hugetlb_fault_mutex_hash(mapping, idx);
|
| 864 | mutex_lock(&hugetlb_fault_mutex_table[hash]);
|
| 865 | hugetlb_vma_lock_read(dst_vma);
|
| 866 |
|
| 867 | err = -ENOMEM;
|
| 868 | dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);
|
| 869 | if (!dst_pte) {
|
| 870 | hugetlb_vma_unlock_read(dst_vma);
|
| 871 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 872 | goto out_unlock;
|
| 873 | }
|
| 874 |
|
| 875 | if (!uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE)) {
|
| 876 | const pte_t ptep = huge_ptep_get(dst_mm, dst_addr, dst_pte);
|
| 877 |
|
| 878 | if (!huge_pte_none(ptep) && !pte_is_uffd_marker(ptep)) {
|
| 879 | err = -EEXIST;
|
| 880 | hugetlb_vma_unlock_read(dst_vma);
|
| 881 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 882 | goto out_unlock;
|
| 883 | }
|
| 884 | }
|
| 885 |
|
| 886 | err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr,
|
| 887 | src_addr, flags, &folio);
|
| 888 |
|
| 889 | hugetlb_vma_unlock_read(dst_vma);
|
| 890 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 891 |
|
| 892 | cond_resched();
|
| 893 |
|
| 894 | if (unlikely(err == -ENOENT)) {
|
| 895 | up_read(&ctx->map_changing_lock);
|
| 896 | uffd_mfill_unlock(dst_vma);
|
| 897 | VM_WARN_ON_ONCE(!folio);
|
| 898 |
|
| 899 | err = copy_folio_from_user(folio,
|
| 900 | (const void __user *)src_addr, true);
|
| 901 | if (unlikely(err)) {
|
| 902 | err = -EFAULT;
|
| 903 | goto out;
|
| 904 | }
|
| 905 |
|
| 906 | dst_vma = NULL;
|
| 907 | goto retry;
|
| 908 | } else
|
| 909 | VM_WARN_ON_ONCE(folio);
|
| 910 |
|
| 911 | if (!err) {
|
| 912 | dst_addr += vma_hpagesize;
|
| 913 | src_addr += vma_hpagesize;
|
| 914 | copied += vma_hpagesize;
|
| 915 |
|
| 916 | if (fatal_signal_pending(current))
|
| 917 | err = -EINTR;
|
| 918 | }
|
| 919 | if (err)
|
| 920 | break;
|
| 921 | }
|
| 922 |
|
| 923 | out_unlock:
|
| 924 | up_read(&ctx->map_changing_lock);
|
| 925 | out_unlock_vma:
|
| 926 | uffd_mfill_unlock(dst_vma);
|
| 927 | out:
|
| 928 | if (folio)
|
| 929 | folio_put(folio);
|
| 930 | VM_WARN_ON_ONCE(copied < 0);
|
| 931 | VM_WARN_ON_ONCE(err > 0);
|
| 932 | VM_WARN_ON_ONCE(!copied && !err);
|
| 933 | return copied ? copied : err;
|
| 934 | }
|
| 935 | #else
|
| 936 |
|
| 937 | extern ssize_t mfill_atomic_hugetlb(struct userfaultfd_ctx *ctx,
|
| 938 | struct vm_area_struct *dst_vma,
|
| 939 | unsigned long dst_start,
|
| 940 | unsigned long src_start,
|
| 941 | unsigned long len,
|
| 942 | uffd_flags_t flags);
|
| 943 | #endif
|
| 944 |
|
| 945 | static __always_inline ssize_t mfill_atomic_pte(struct mfill_state *state)
|
| 946 | {
|
| 947 | uffd_flags_t flags = state->flags;
|
| 948 |
|
| 949 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
|
| 950 | return mfill_atomic_pte_continue(state);
|
| 951 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON))
|
| 952 | return mfill_atomic_pte_poison(state);
|
| 953 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY))
|
| 954 | return mfill_atomic_pte_copy(state);
|
| 955 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_ZEROPAGE))
|
| 956 | return mfill_atomic_pte_zeropage(state);
|
| 957 |
|
| 958 | VM_WARN_ONCE(1, "Unknown UFFDIO operation, flags: %x", flags);
|
| 959 | return -EOPNOTSUPP;
|
| 960 | }
|
| 961 |
|
| 962 | static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
|
| 963 | unsigned long dst_start,
|
| 964 | unsigned long src_start,
|
| 965 | unsigned long len,
|
| 966 | uffd_flags_t flags)
|
| 967 | {
|
| 968 | struct mfill_state state = (struct mfill_state){
|
| 969 | .ctx = ctx,
|
| 970 | .dst_start = dst_start,
|
| 971 | .src_start = src_start,
|
| 972 | .flags = flags,
|
| 973 | .len = len,
|
| 974 | .src_addr = src_start,
|
| 975 | .dst_addr = dst_start,
|
| 976 | };
|
| 977 | long copied = 0;
|
| 978 | ssize_t err;
|
| 979 |
|
| 980 | |
| 981 | * Sanitize the command parameters:
|
| 982 |
|
| 983 | VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK);
|
| 984 | VM_WARN_ON_ONCE(len & ~PAGE_MASK);
|
| 985 |
|
| 986 |
|
| 987 | VM_WARN_ON_ONCE(src_start + len <= src_start);
|
| 988 | VM_WARN_ON_ONCE(dst_start + len <= dst_start);
|
| 989 |
|
| 990 | err = mfill_get_vma(&state);
|
| 991 | if (err)
|
| 992 | goto out;
|
| 993 |
|
| 994 | |
| 995 | * If this is a HUGETLB vma, pass off to appropriate routine
|
| 996 |
|
| 997 | if (is_vm_hugetlb_page(state.vma))
|
| 998 | return mfill_atomic_hugetlb(ctx, state.vma, dst_start,
|
| 999 | src_start, len, flags);
|
| 1000 |
|
| 1001 | while (state.src_addr < src_start + len) {
|
| 1002 | VM_WARN_ON_ONCE(state.dst_addr >= dst_start + len);
|
| 1003 |
|
| 1004 | err = mfill_establish_pmd(&state);
|
| 1005 | if (err)
|
| 1006 | break;
|
| 1007 |
|
| 1008 | |
| 1009 | * For shmem mappings, khugepaged is allowed to remove page
|
| 1010 | * tables under us; pte_offset_map_lock() will deal with that.
|
| 1011 |
|
| 1012 |
|
| 1013 | err = mfill_atomic_pte(&state);
|
| 1014 | cond_resched();
|
| 1015 |
|
| 1016 | if (!err) {
|
| 1017 | state.dst_addr += PAGE_SIZE;
|
| 1018 | state.src_addr += PAGE_SIZE;
|
| 1019 | copied += PAGE_SIZE;
|
| 1020 |
|
| 1021 | if (fatal_signal_pending(current))
|
| 1022 | err = -EINTR;
|
| 1023 | }
|
| 1024 | if (err)
|
| 1025 | break;
|
| 1026 | }
|
| 1027 |
|
| 1028 | mfill_put_vma(&state);
|
| 1029 | out:
|
| 1030 | VM_WARN_ON_ONCE(copied < 0);
|
| 1031 | VM_WARN_ON_ONCE(err > 0);
|
| 1032 | VM_WARN_ON_ONCE(!copied && !err);
|
| 1033 | return copied ? copied : err;
|
| 1034 | }
|
| 1035 |
|
| 1036 | static ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start,
|
| 1037 | unsigned long src_start, unsigned long len,
|
| 1038 | uffd_flags_t flags)
|
| 1039 | {
|
| 1040 | return mfill_atomic(ctx, dst_start, src_start, len,
|
| 1041 | uffd_flags_set_mode(flags, MFILL_ATOMIC_COPY));
|
| 1042 | }
|
| 1043 |
|
| 1044 | static ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx,
|
| 1045 | unsigned long start,
|
| 1046 | unsigned long len)
|
| 1047 | {
|
| 1048 | return mfill_atomic(ctx, start, 0, len,
|
| 1049 | uffd_flags_set_mode(0, MFILL_ATOMIC_ZEROPAGE));
|
| 1050 | }
|
| 1051 |
|
| 1052 | static ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long start,
|
| 1053 | unsigned long len, uffd_flags_t flags)
|
| 1054 | {
|
| 1055 |
|
| 1056 | |
| 1057 | * A caller might reasonably assume that UFFDIO_CONTINUE contains an
|
| 1058 | * smp_wmb() to ensure that any writes to the about-to-be-mapped page by
|
| 1059 | * the thread doing the UFFDIO_CONTINUE are guaranteed to be visible to
|
| 1060 | * subsequent loads from the page through the newly mapped address range.
|
| 1061 |
|
| 1062 | smp_wmb();
|
| 1063 |
|
| 1064 | return mfill_atomic(ctx, start, 0, len,
|
| 1065 | uffd_flags_set_mode(flags, MFILL_ATOMIC_CONTINUE));
|
| 1066 | }
|
| 1067 |
|
| 1068 | static ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start,
|
| 1069 | unsigned long len, uffd_flags_t flags)
|
| 1070 | {
|
| 1071 | return mfill_atomic(ctx, start, 0, len,
|
| 1072 | uffd_flags_set_mode(flags, MFILL_ATOMIC_POISON));
|
| 1073 | }
|
| 1074 |
|
| 1075 | long uffd_wp_range(struct vm_area_struct *dst_vma,
|
| 1076 | unsigned long start, unsigned long len, bool enable_wp)
|
| 1077 | {
|
| 1078 | unsigned int mm_cp_flags;
|
| 1079 | struct mmu_gather tlb;
|
| 1080 | long ret;
|
| 1081 |
|
| 1082 | VM_WARN_ONCE(start < dst_vma->vm_start || start + len > dst_vma->vm_end,
|
| 1083 | "The address range exceeds VMA boundary.\n");
|
| 1084 | if (enable_wp)
|
| 1085 | mm_cp_flags = MM_CP_UFFD_WP;
|
| 1086 | else
|
| 1087 | mm_cp_flags = MM_CP_UFFD_WP_RESOLVE;
|
| 1088 |
|
| 1089 | |
| 1090 | * vma->vm_page_prot already reflects that uffd-wp is enabled for this
|
| 1091 | * VMA (see userfaultfd_set_vm_flags()) and that all PTEs are supposed
|
| 1092 | * to be write-protected as default whenever protection changes.
|
| 1093 | * Try upgrading write permissions manually.
|
| 1094 |
|
| 1095 | if (!enable_wp && vma_wants_manual_pte_write_upgrade(dst_vma))
|
| 1096 | mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
|
| 1097 | tlb_gather_mmu(&tlb, dst_vma->vm_mm);
|
| 1098 | ret = change_protection(&tlb, dst_vma, start, start + len, mm_cp_flags);
|
| 1099 | tlb_finish_mmu(&tlb);
|
| 1100 |
|
| 1101 | return ret;
|
| 1102 | }
|
| 1103 |
|
| 1104 | static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
|
| 1105 | unsigned long len, bool enable_wp)
|
| 1106 | {
|
| 1107 | struct mm_struct *dst_mm = ctx->mm;
|
| 1108 | unsigned long end = start + len;
|
| 1109 | unsigned long _start, _end;
|
| 1110 | struct vm_area_struct *dst_vma;
|
| 1111 | unsigned long page_mask;
|
| 1112 | long err;
|
| 1113 | VMA_ITERATOR(vmi, dst_mm, start);
|
| 1114 |
|
| 1115 | |
| 1116 | * Sanitize the command parameters:
|
| 1117 |
|
| 1118 | VM_WARN_ON_ONCE(start & ~PAGE_MASK);
|
| 1119 | VM_WARN_ON_ONCE(len & ~PAGE_MASK);
|
| 1120 |
|
| 1121 |
|
| 1122 | VM_WARN_ON_ONCE(start + len <= start);
|
| 1123 |
|
| 1124 | mmap_read_lock(dst_mm);
|
| 1125 |
|
| 1126 | |
| 1127 | * If memory mappings are changing because of non-cooperative
|
| 1128 | * operation (e.g. mremap) running in parallel, bail out and
|
| 1129 | * request the user to retry later
|
| 1130 |
|
| 1131 | down_read(&ctx->map_changing_lock);
|
| 1132 | err = -EAGAIN;
|
| 1133 | if (atomic_read(&ctx->mmap_changing))
|
| 1134 | goto out_unlock;
|
| 1135 |
|
| 1136 | err = -ENOENT;
|
| 1137 | for_each_vma_range(vmi, dst_vma, end) {
|
| 1138 |
|
| 1139 | if (!userfaultfd_wp(dst_vma)) {
|
| 1140 | err = -ENOENT;
|
| 1141 | break;
|
| 1142 | }
|
| 1143 |
|
| 1144 | if (is_vm_hugetlb_page(dst_vma)) {
|
| 1145 | err = -EINVAL;
|
| 1146 | page_mask = vma_kernel_pagesize(dst_vma) - 1;
|
| 1147 | if ((start & page_mask) || (len & page_mask))
|
| 1148 | break;
|
| 1149 | }
|
| 1150 |
|
| 1151 | _start = max(dst_vma->vm_start, start);
|
| 1152 | _end = min(dst_vma->vm_end, end);
|
| 1153 |
|
| 1154 | err = uffd_wp_range(dst_vma, _start, _end - _start, enable_wp);
|
| 1155 |
|
| 1156 |
|
| 1157 | if (err < 0)
|
| 1158 | break;
|
| 1159 | err = 0;
|
| 1160 | }
|
| 1161 | out_unlock:
|
| 1162 | up_read(&ctx->map_changing_lock);
|
| 1163 | mmap_read_unlock(dst_mm);
|
| 1164 | return err;
|
| 1165 | }
|
| 1166 |
|
| 1167 | int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
|
| 1168 | unsigned long len, bool enable_rwp)
|
| 1169 | {
|
| 1170 | struct mm_struct *dst_mm = ctx->mm;
|
| 1171 | unsigned long end = start + len;
|
| 1172 | struct vm_area_struct *dst_vma;
|
| 1173 | unsigned int mm_cp_flags;
|
| 1174 | struct mmu_gather tlb;
|
| 1175 | bool found = false;
|
| 1176 | VMA_ITERATOR(vmi, dst_mm, start);
|
| 1177 |
|
| 1178 | VM_WARN_ON_ONCE(start & ~PAGE_MASK);
|
| 1179 | VM_WARN_ON_ONCE(len & ~PAGE_MASK);
|
| 1180 | VM_WARN_ON_ONCE(start + len <= start);
|
| 1181 |
|
| 1182 | guard(mmap_read_lock)(dst_mm);
|
| 1183 | guard(rwsem_read)(&ctx->map_changing_lock);
|
| 1184 |
|
| 1185 | if (atomic_read(&ctx->mmap_changing))
|
| 1186 | return -EAGAIN;
|
| 1187 |
|
| 1188 | if (enable_rwp)
|
| 1189 | mm_cp_flags = MM_CP_UFFD_RWP;
|
| 1190 | else
|
| 1191 | mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE;
|
| 1192 |
|
| 1193 | |
| 1194 | * Pre-scan the range: validate every spanned VMA before applying
|
| 1195 | * any change_protection() so a partial failure cannot leave the
|
| 1196 | * process with only a prefix of the range re-protected.
|
| 1197 |
|
| 1198 | for_each_vma_range(vmi, dst_vma, end) {
|
| 1199 | if (!userfaultfd_rwp(dst_vma))
|
| 1200 | return -ENOENT;
|
| 1201 |
|
| 1202 | if (is_vm_hugetlb_page(dst_vma)) {
|
| 1203 | unsigned long page_mask;
|
| 1204 |
|
| 1205 | page_mask = vma_kernel_pagesize(dst_vma) - 1;
|
| 1206 | if ((start & page_mask) || (len & page_mask))
|
| 1207 | return -EINVAL;
|
| 1208 | }
|
| 1209 | found = true;
|
| 1210 | }
|
| 1211 | if (!found)
|
| 1212 | return -ENOENT;
|
| 1213 |
|
| 1214 | vma_iter_set(&vmi, start);
|
| 1215 | tlb_gather_mmu(&tlb, dst_mm);
|
| 1216 | for_each_vma_range(vmi, dst_vma, end) {
|
| 1217 | unsigned long vma_start = max(dst_vma->vm_start, start);
|
| 1218 | unsigned long vma_end = min(dst_vma->vm_end, end);
|
| 1219 | unsigned int flags = mm_cp_flags;
|
| 1220 |
|
| 1221 | |
| 1222 | * On resolve, try to upgrade writability per-VMA --
|
| 1223 | * MM_CP_TRY_CHANGE_WRITABLE WARNs in
|
| 1224 | * maybe_change_pte_writable() if the VMA is not VM_WRITE,
|
| 1225 | * and RWP can be registered on PROT_READ-only mappings.
|
| 1226 |
|
| 1227 | if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma))
|
| 1228 | flags |= MM_CP_TRY_CHANGE_WRITABLE;
|
| 1229 |
|
| 1230 | change_protection(&tlb, dst_vma, vma_start, vma_end, flags);
|
| 1231 | }
|
| 1232 | tlb_finish_mmu(&tlb);
|
| 1233 |
|
| 1234 | return 0;
|
| 1235 | }
|
| 1236 |
|
| 1237 | void double_pt_lock(spinlock_t *ptl1,
|
| 1238 | spinlock_t *ptl2)
|
| 1239 | __acquires(ptl1)
|
| 1240 | __acquires(ptl2)
|
| 1241 | {
|
| 1242 | if (ptl1 > ptl2)
|
| 1243 | swap(ptl1, ptl2);
|
| 1244 |
|
| 1245 | spin_lock(ptl1);
|
| 1246 | if (ptl1 != ptl2)
|
| 1247 | spin_lock_nested(ptl2, SINGLE_DEPTH_NESTING);
|
| 1248 | else
|
| 1249 | __acquire(ptl2);
|
| 1250 | }
|
| 1251 |
|
| 1252 | void double_pt_unlock(spinlock_t *ptl1,
|
| 1253 | spinlock_t *ptl2)
|
| 1254 | __releases(ptl1)
|
| 1255 | __releases(ptl2)
|
| 1256 | {
|
| 1257 | spin_unlock(ptl1);
|
| 1258 | if (ptl1 != ptl2)
|
| 1259 | spin_unlock(ptl2);
|
| 1260 | else
|
| 1261 | __release(ptl2);
|
| 1262 | }
|
| 1263 |
|
| 1264 | static inline bool is_pte_pages_stable(pte_t *dst_pte, pte_t *src_pte,
|
| 1265 | pte_t orig_dst_pte, pte_t orig_src_pte,
|
| 1266 | pmd_t *dst_pmd, pmd_t dst_pmdval)
|
| 1267 | {
|
| 1268 | return pte_same(ptep_get(src_pte), orig_src_pte) &&
|
| 1269 | pte_same(ptep_get(dst_pte), orig_dst_pte) &&
|
| 1270 | pmd_same(dst_pmdval, pmdp_get_lockless(dst_pmd));
|
| 1271 | }
|
| 1272 |
|
| 1273 | |
| 1274 | * Checks if the two ptes and the corresponding folio are eligible for batched
|
| 1275 | * move. If so, then returns pointer to the locked folio. Otherwise, returns NULL.
|
| 1276 | *
|
| 1277 | * NOTE: folio's reference is not required as the whole operation is within
|
| 1278 | * PTL's critical section.
|
| 1279 |
|
| 1280 | static struct folio *check_ptes_for_batched_move(struct vm_area_struct *src_vma,
|
| 1281 | unsigned long src_addr,
|
| 1282 | pte_t *src_pte, pte_t *dst_pte)
|
| 1283 | {
|
| 1284 | pte_t orig_dst_pte, orig_src_pte;
|
| 1285 | struct folio *folio;
|
| 1286 |
|
| 1287 | orig_dst_pte = ptep_get(dst_pte);
|
| 1288 | if (!pte_none(orig_dst_pte))
|
| 1289 | return NULL;
|
| 1290 |
|
| 1291 | orig_src_pte = ptep_get(src_pte);
|
| 1292 | if (!pte_present(orig_src_pte) || is_zero_pfn(pte_pfn(orig_src_pte)))
|
| 1293 | return NULL;
|
| 1294 |
|
| 1295 | folio = vm_normal_folio(src_vma, src_addr, orig_src_pte);
|
| 1296 | if (!folio || !folio_trylock(folio))
|
| 1297 | return NULL;
|
| 1298 | if (!PageAnonExclusive(&folio->page) || folio_test_large(folio)) {
|
| 1299 | folio_unlock(folio);
|
| 1300 | return NULL;
|
| 1301 | }
|
| 1302 | return folio;
|
| 1303 | }
|
| 1304 |
|
| 1305 | |
| 1306 | * Moves src folios to dst in a batch as long as they are not large, and can
|
| 1307 | * successfully take the lock via folio_trylock().
|
| 1308 |
|
| 1309 | static long move_present_ptes(struct mm_struct *mm,
|
| 1310 | struct vm_area_struct *dst_vma,
|
| 1311 | struct vm_area_struct *src_vma,
|
| 1312 | unsigned long dst_addr, unsigned long src_addr,
|
| 1313 | pte_t *dst_pte, pte_t *src_pte,
|
| 1314 | pte_t orig_dst_pte, pte_t orig_src_pte,
|
| 1315 | pmd_t *dst_pmd, pmd_t dst_pmdval,
|
| 1316 | spinlock_t *dst_ptl, spinlock_t *src_ptl,
|
| 1317 | struct folio **first_src_folio, unsigned long len)
|
| 1318 | {
|
| 1319 | int err = 0;
|
| 1320 | struct folio *src_folio = *first_src_folio;
|
| 1321 | unsigned long src_start = src_addr;
|
| 1322 | unsigned long src_end;
|
| 1323 |
|
| 1324 | len = pmd_addr_end(dst_addr, dst_addr + len) - dst_addr;
|
| 1325 | src_end = pmd_addr_end(src_addr, src_addr + len);
|
| 1326 | flush_cache_range(src_vma, src_addr, src_end);
|
| 1327 | double_pt_lock(dst_ptl, src_ptl);
|
| 1328 |
|
| 1329 | if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte,
|
| 1330 | dst_pmd, dst_pmdval)) {
|
| 1331 | err = -EAGAIN;
|
| 1332 | goto out;
|
| 1333 | }
|
| 1334 | if (folio_test_large(src_folio) ||
|
| 1335 | folio_maybe_dma_pinned(src_folio) ||
|
| 1336 | !PageAnonExclusive(&src_folio->page)) {
|
| 1337 | err = -EBUSY;
|
| 1338 | goto out;
|
| 1339 | }
|
| 1340 |
|
| 1341 | folio_put(*first_src_folio);
|
| 1342 | *first_src_folio = NULL;
|
| 1343 | lazy_mmu_mode_enable();
|
| 1344 |
|
| 1345 | while (true) {
|
| 1346 | orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
|
| 1347 |
|
| 1348 | if (folio_maybe_dma_pinned(src_folio)) {
|
| 1349 | set_pte_at(mm, src_addr, src_pte, orig_src_pte);
|
| 1350 | err = -EBUSY;
|
| 1351 | break;
|
| 1352 | }
|
| 1353 |
|
| 1354 | folio_move_anon_rmap(src_folio, dst_vma);
|
| 1355 | src_folio->index = linear_anon_page_index(dst_vma, dst_addr);
|
| 1356 |
|
| 1357 | orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot);
|
| 1358 |
|
| 1359 | if (pgtable_supports_soft_dirty())
|
| 1360 | orig_dst_pte = pte_mksoft_dirty(orig_dst_pte);
|
| 1361 | if (pte_dirty(orig_src_pte))
|
| 1362 | orig_dst_pte = pte_mkdirty(orig_dst_pte);
|
| 1363 | orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma);
|
| 1364 |
|
| 1365 |
|
| 1366 | if (userfaultfd_rwp(dst_vma)) {
|
| 1367 | orig_dst_pte = pte_modify(orig_dst_pte, PAGE_NONE);
|
| 1368 | orig_dst_pte = pte_mkuffd(orig_dst_pte);
|
| 1369 | }
|
| 1370 |
|
| 1371 | set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte);
|
| 1372 |
|
| 1373 | src_addr += PAGE_SIZE;
|
| 1374 | if (src_addr == src_end)
|
| 1375 | break;
|
| 1376 | dst_addr += PAGE_SIZE;
|
| 1377 | dst_pte++;
|
| 1378 | src_pte++;
|
| 1379 |
|
| 1380 | folio_unlock(src_folio);
|
| 1381 | src_folio = check_ptes_for_batched_move(src_vma, src_addr,
|
| 1382 | src_pte, dst_pte);
|
| 1383 | if (!src_folio)
|
| 1384 | break;
|
| 1385 | }
|
| 1386 |
|
| 1387 | lazy_mmu_mode_disable();
|
| 1388 | if (src_addr > src_start)
|
| 1389 | flush_tlb_range(src_vma, src_start, src_addr);
|
| 1390 |
|
| 1391 | if (src_folio)
|
| 1392 | folio_unlock(src_folio);
|
| 1393 | out:
|
| 1394 | double_pt_unlock(dst_ptl, src_ptl);
|
| 1395 | return src_addr > src_start ? src_addr - src_start : err;
|
| 1396 | }
|
| 1397 |
|
| 1398 | static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,
|
| 1399 | unsigned long dst_addr, unsigned long src_addr,
|
| 1400 | pte_t *dst_pte, pte_t *src_pte,
|
| 1401 | pte_t orig_dst_pte, pte_t orig_src_pte,
|
| 1402 | pmd_t *dst_pmd, pmd_t dst_pmdval,
|
| 1403 | spinlock_t *dst_ptl, spinlock_t *src_ptl,
|
| 1404 | struct folio *src_folio,
|
| 1405 | struct swap_info_struct *si, swp_entry_t entry)
|
| 1406 | {
|
| 1407 | |
| 1408 | * Check if the folio still belongs to the target swap entry after
|
| 1409 | * acquiring the lock. Folio can be freed in the swap cache while
|
| 1410 | * not locked.
|
| 1411 |
|
| 1412 | if (src_folio && unlikely(!folio_test_swapcache(src_folio) ||
|
| 1413 | entry.val != src_folio->swap.val))
|
| 1414 | return -EAGAIN;
|
| 1415 |
|
| 1416 | double_pt_lock(dst_ptl, src_ptl);
|
| 1417 |
|
| 1418 | if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte,
|
| 1419 | dst_pmd, dst_pmdval)) {
|
| 1420 | double_pt_unlock(dst_ptl, src_ptl);
|
| 1421 | return -EAGAIN;
|
| 1422 | }
|
| 1423 |
|
| 1424 | |
| 1425 | * The src_folio resides in the swapcache, requiring an update to its
|
| 1426 | * index and mapping to align with the dst_vma, where a swap-in may
|
| 1427 | * occur and hit the swapcache after moving the PTE.
|
| 1428 |
|
| 1429 | if (src_folio) {
|
| 1430 | folio_move_anon_rmap(src_folio, dst_vma);
|
| 1431 | src_folio->index = linear_anon_page_index(dst_vma, dst_addr);
|
| 1432 | } else {
|
| 1433 | |
| 1434 | * Check if the swap entry is cached after acquiring the src_pte
|
| 1435 | * lock. Otherwise, we might miss a newly loaded swap cache folio.
|
| 1436 | *
|
| 1437 | * We are trying to catch newly added swap cache, the only possible case is
|
| 1438 | * when a folio is swapped in and out again staying in swap cache, using the
|
| 1439 | * same entry before the PTE check above. The PTL is acquired and released
|
| 1440 | * twice, each time after updating the swap table. So holding
|
| 1441 | * the PTL here ensures we see the updated value.
|
| 1442 |
|
| 1443 | if (swap_cache_has_folio(entry)) {
|
| 1444 | double_pt_unlock(dst_ptl, src_ptl);
|
| 1445 | return -EAGAIN;
|
| 1446 | }
|
| 1447 | }
|
| 1448 |
|
| 1449 | orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
|
| 1450 | if (pgtable_supports_soft_dirty())
|
| 1451 | orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte);
|
| 1452 |
|
| 1453 | if (userfaultfd_rwp(dst_vma))
|
| 1454 | orig_src_pte = pte_swp_mkuffd(orig_src_pte);
|
| 1455 | set_pte_at(mm, dst_addr, dst_pte, orig_src_pte);
|
| 1456 | double_pt_unlock(dst_ptl, src_ptl);
|
| 1457 |
|
| 1458 | return PAGE_SIZE;
|
| 1459 | }
|
| 1460 |
|
| 1461 | static int move_zeropage_pte(struct mm_struct *mm,
|
| 1462 | struct vm_area_struct *dst_vma,
|
| 1463 | struct vm_area_struct *src_vma,
|
| 1464 | unsigned long dst_addr, unsigned long src_addr,
|
| 1465 | pte_t *dst_pte, pte_t *src_pte,
|
| 1466 | pte_t orig_dst_pte, pte_t orig_src_pte,
|
| 1467 | pmd_t *dst_pmd, pmd_t dst_pmdval,
|
| 1468 | spinlock_t *dst_ptl, spinlock_t *src_ptl)
|
| 1469 | {
|
| 1470 | pte_t zero_pte;
|
| 1471 |
|
| 1472 | double_pt_lock(dst_ptl, src_ptl);
|
| 1473 | if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst_pte, orig_src_pte,
|
| 1474 | dst_pmd, dst_pmdval)) {
|
| 1475 | double_pt_unlock(dst_ptl, src_ptl);
|
| 1476 | return -EAGAIN;
|
| 1477 | }
|
| 1478 |
|
| 1479 | zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr),
|
| 1480 | dst_vma->vm_page_prot));
|
| 1481 |
|
| 1482 |
|
| 1483 | if (userfaultfd_rwp(dst_vma)) {
|
| 1484 | zero_pte = pte_modify(zero_pte, PAGE_NONE);
|
| 1485 | zero_pte = pte_mkuffd(zero_pte);
|
| 1486 | }
|
| 1487 |
|
| 1488 | ptep_clear_flush(src_vma, src_addr, src_pte);
|
| 1489 | set_pte_at(mm, dst_addr, dst_pte, zero_pte);
|
| 1490 | double_pt_unlock(dst_ptl, src_ptl);
|
| 1491 |
|
| 1492 | return PAGE_SIZE;
|
| 1493 | }
|
| 1494 |
|
| 1495 |
|
| 1496 | |
| 1497 | * The mmap_lock for reading is held by the caller. Just move the page(s)
|
| 1498 | * from src_pmd to dst_pmd if possible, and return number of bytes moved.
|
| 1499 | * On failure, an error code is returned.
|
| 1500 |
|
| 1501 | static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd,
|
| 1502 | struct vm_area_struct *dst_vma,
|
| 1503 | struct vm_area_struct *src_vma,
|
| 1504 | unsigned long dst_addr, unsigned long src_addr,
|
| 1505 | unsigned long len, __u64 mode)
|
| 1506 | {
|
| 1507 | struct swap_info_struct *si = NULL;
|
| 1508 | pte_t orig_src_pte, orig_dst_pte;
|
| 1509 | pte_t src_folio_pte;
|
| 1510 | spinlock_t *src_ptl, *dst_ptl;
|
| 1511 | pte_t *src_pte = NULL;
|
| 1512 | pte_t *dst_pte = NULL;
|
| 1513 | pmd_t dummy_pmdval;
|
| 1514 | pmd_t dst_pmdval;
|
| 1515 | struct folio *src_folio = NULL;
|
| 1516 | struct mmu_notifier_range range;
|
| 1517 | long ret = 0;
|
| 1518 |
|
| 1519 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
|
| 1520 | src_addr, src_addr + len);
|
| 1521 | mmu_notifier_invalidate_range_start(&range);
|
| 1522 | retry:
|
| 1523 | |
| 1524 | * Use the maywrite version to indicate that dst_pte will be modified,
|
| 1525 | * since dst_pte needs to be none, the subsequent pte_same() check
|
| 1526 | * cannot prevent the dst_pte page from being freed concurrently, so we
|
| 1527 | * also need to obtain dst_pmdval and recheck pmd_same() later.
|
| 1528 |
|
| 1529 | dst_pte = pte_offset_map_rw_nolock(mm, dst_pmd, dst_addr, &dst_pmdval,
|
| 1530 | &dst_ptl);
|
| 1531 |
|
| 1532 |
|
| 1533 | if (unlikely(!dst_pte)) {
|
| 1534 | ret = -EAGAIN;
|
| 1535 | goto out;
|
| 1536 | }
|
| 1537 |
|
| 1538 | |
| 1539 | * Unlike dst_pte, the subsequent pte_same() check can ensure the
|
| 1540 | * stability of the src_pte page, so there is no need to get pmdval,
|
| 1541 | * just pass a dummy variable to it.
|
| 1542 |
|
| 1543 | src_pte = pte_offset_map_rw_nolock(mm, src_pmd, src_addr, &dummy_pmdval,
|
| 1544 | &src_ptl);
|
| 1545 |
|
| 1546 | |
| 1547 | * We held the mmap_lock for reading so MADV_DONTNEED
|
| 1548 | * can zap transparent huge pages under us, or the
|
| 1549 | * transparent huge page fault can establish new
|
| 1550 | * transparent huge pages under us.
|
| 1551 |
|
| 1552 | if (unlikely(!src_pte)) {
|
| 1553 | ret = -EAGAIN;
|
| 1554 | goto out;
|
| 1555 | }
|
| 1556 |
|
| 1557 |
|
| 1558 | if (pmd_none(*dst_pmd) || pmd_none(*src_pmd) ||
|
| 1559 | pmd_trans_huge(*dst_pmd) || pmd_trans_huge(*src_pmd)) {
|
| 1560 | ret = -EINVAL;
|
| 1561 | goto out;
|
| 1562 | }
|
| 1563 |
|
| 1564 | spin_lock(dst_ptl);
|
| 1565 | orig_dst_pte = ptep_get(dst_pte);
|
| 1566 | spin_unlock(dst_ptl);
|
| 1567 | if (!pte_none(orig_dst_pte)) {
|
| 1568 | ret = -EEXIST;
|
| 1569 | goto out;
|
| 1570 | }
|
| 1571 |
|
| 1572 | spin_lock(src_ptl);
|
| 1573 | orig_src_pte = ptep_get(src_pte);
|
| 1574 | spin_unlock(src_ptl);
|
| 1575 | if (pte_none(orig_src_pte)) {
|
| 1576 | if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES))
|
| 1577 | ret = -ENOENT;
|
| 1578 | else
|
| 1579 | ret = PAGE_SIZE;
|
| 1580 | goto out;
|
| 1581 | }
|
| 1582 |
|
| 1583 |
|
| 1584 | if (src_folio && unlikely(!pte_same(src_folio_pte, orig_src_pte))) {
|
| 1585 | ret = -EAGAIN;
|
| 1586 | goto out;
|
| 1587 | }
|
| 1588 |
|
| 1589 | if (pte_present(orig_src_pte)) {
|
| 1590 | if (is_zero_pfn(pte_pfn(orig_src_pte))) {
|
| 1591 | ret = move_zeropage_pte(mm, dst_vma, src_vma,
|
| 1592 | dst_addr, src_addr, dst_pte, src_pte,
|
| 1593 | orig_dst_pte, orig_src_pte,
|
| 1594 | dst_pmd, dst_pmdval, dst_ptl, src_ptl);
|
| 1595 | goto out;
|
| 1596 | }
|
| 1597 |
|
| 1598 | |
| 1599 | * Pin and lock source folio. Since we are in RCU read section,
|
| 1600 | * we can't block, so on contention have to unmap the ptes,
|
| 1601 | * obtain the lock and retry.
|
| 1602 |
|
| 1603 | if (!src_folio) {
|
| 1604 | struct folio *folio;
|
| 1605 | bool locked;
|
| 1606 |
|
| 1607 | |
| 1608 | * Pin the page while holding the lock to be sure the
|
| 1609 | * page isn't freed under us
|
| 1610 |
|
| 1611 | spin_lock(src_ptl);
|
| 1612 | if (!pte_same(orig_src_pte, ptep_get(src_pte))) {
|
| 1613 | spin_unlock(src_ptl);
|
| 1614 | ret = -EAGAIN;
|
| 1615 | goto out;
|
| 1616 | }
|
| 1617 |
|
| 1618 | folio = vm_normal_folio(src_vma, src_addr, orig_src_pte);
|
| 1619 | if (!folio || !PageAnonExclusive(&folio->page)) {
|
| 1620 | spin_unlock(src_ptl);
|
| 1621 | ret = -EBUSY;
|
| 1622 | goto out;
|
| 1623 | }
|
| 1624 |
|
| 1625 | locked = folio_trylock(folio);
|
| 1626 | |
| 1627 | * We avoid waiting for folio lock with a raised
|
| 1628 | * refcount for large folios because extra refcounts
|
| 1629 | * will result in split_folio() failing later and
|
| 1630 | * retrying. If multiple tasks are trying to move a
|
| 1631 | * large folio we can end up livelocking.
|
| 1632 |
|
| 1633 | if (!locked && folio_test_large(folio)) {
|
| 1634 | spin_unlock(src_ptl);
|
| 1635 | ret = -EAGAIN;
|
| 1636 | goto out;
|
| 1637 | }
|
| 1638 |
|
| 1639 | folio_get(folio);
|
| 1640 | src_folio = folio;
|
| 1641 | src_folio_pte = orig_src_pte;
|
| 1642 | spin_unlock(src_ptl);
|
| 1643 |
|
| 1644 | if (!locked) {
|
| 1645 | pte_unmap(src_pte);
|
| 1646 | pte_unmap(dst_pte);
|
| 1647 | src_pte = dst_pte = NULL;
|
| 1648 |
|
| 1649 | folio_lock(src_folio);
|
| 1650 | goto retry;
|
| 1651 | }
|
| 1652 |
|
| 1653 | if (WARN_ON_ONCE(!folio_test_anon(src_folio))) {
|
| 1654 | ret = -EBUSY;
|
| 1655 | goto out;
|
| 1656 | }
|
| 1657 | }
|
| 1658 |
|
| 1659 |
|
| 1660 | if (folio_test_large(src_folio)) {
|
| 1661 |
|
| 1662 | pte_unmap(src_pte);
|
| 1663 | pte_unmap(dst_pte);
|
| 1664 | src_pte = dst_pte = NULL;
|
| 1665 | ret = split_folio(src_folio);
|
| 1666 | if (ret)
|
| 1667 | goto out;
|
| 1668 |
|
| 1669 | folio_unlock(src_folio);
|
| 1670 | folio_put(src_folio);
|
| 1671 | src_folio = NULL;
|
| 1672 | goto retry;
|
| 1673 | }
|
| 1674 |
|
| 1675 | ret = move_present_ptes(mm, dst_vma, src_vma,
|
| 1676 | dst_addr, src_addr, dst_pte, src_pte,
|
| 1677 | orig_dst_pte, orig_src_pte, dst_pmd,
|
| 1678 | dst_pmdval, dst_ptl, src_ptl, &src_folio,
|
| 1679 | len);
|
| 1680 | } else {
|
| 1681 | struct folio *folio = NULL;
|
| 1682 | const softleaf_t entry = softleaf_from_pte(orig_src_pte);
|
| 1683 |
|
| 1684 | if (softleaf_is_migration(entry)) {
|
| 1685 | pte_unmap(src_pte);
|
| 1686 | pte_unmap(dst_pte);
|
| 1687 | src_pte = dst_pte = NULL;
|
| 1688 | migration_entry_wait(mm, src_pmd, src_addr);
|
| 1689 |
|
| 1690 | ret = -EAGAIN;
|
| 1691 | goto out;
|
| 1692 | } else if (!softleaf_is_swap(entry)) {
|
| 1693 | ret = -EFAULT;
|
| 1694 | goto out;
|
| 1695 | }
|
| 1696 |
|
| 1697 | if (!pte_swp_exclusive(orig_src_pte)) {
|
| 1698 | ret = -EBUSY;
|
| 1699 | goto out;
|
| 1700 | }
|
| 1701 |
|
| 1702 | si = get_swap_device(entry);
|
| 1703 | if (unlikely(!si)) {
|
| 1704 | ret = -EAGAIN;
|
| 1705 | goto out;
|
| 1706 | }
|
| 1707 | |
| 1708 | * Verify the existence of the swapcache. If present, the folio's
|
| 1709 | * index and mapping must be updated even when the PTE is a swap
|
| 1710 | * entry. The anon_vma lock is not taken during this process since
|
| 1711 | * the folio has already been unmapped, and the swap entry is
|
| 1712 | * exclusive, preventing rmap walks.
|
| 1713 | *
|
| 1714 | * For large folios, return -EBUSY immediately, as split_folio()
|
| 1715 | * also returns -EBUSY when attempting to split unmapped large
|
| 1716 | * folios in the swapcache. This issue needs to be resolved
|
| 1717 | * separately to allow proper handling.
|
| 1718 |
|
| 1719 | if (!src_folio)
|
| 1720 | folio = swap_cache_get_folio(entry);
|
| 1721 | if (folio) {
|
| 1722 | if (folio_test_large(folio)) {
|
| 1723 | ret = -EBUSY;
|
| 1724 | folio_put(folio);
|
| 1725 | goto out;
|
| 1726 | }
|
| 1727 | src_folio = folio;
|
| 1728 | src_folio_pte = orig_src_pte;
|
| 1729 | if (!folio_trylock(src_folio)) {
|
| 1730 | pte_unmap(src_pte);
|
| 1731 | pte_unmap(dst_pte);
|
| 1732 | src_pte = dst_pte = NULL;
|
| 1733 | put_swap_device(si);
|
| 1734 | si = NULL;
|
| 1735 |
|
| 1736 | folio_lock(src_folio);
|
| 1737 | goto retry;
|
| 1738 | }
|
| 1739 | }
|
| 1740 | ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte,
|
| 1741 | orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval,
|
| 1742 | dst_ptl, src_ptl, src_folio, si, entry);
|
| 1743 | }
|
| 1744 |
|
| 1745 | out:
|
| 1746 | if (src_folio) {
|
| 1747 | folio_unlock(src_folio);
|
| 1748 | folio_put(src_folio);
|
| 1749 | }
|
| 1750 | |
| 1751 | * Unmap in reverse order (LIFO) to maintain proper kmap_local
|
| 1752 | * index ordering when CONFIG_HIGHPTE is enabled. We mapped dst_pte
|
| 1753 | * first, then src_pte, so we must unmap src_pte first, then dst_pte.
|
| 1754 |
|
| 1755 | if (src_pte)
|
| 1756 | pte_unmap(src_pte);
|
| 1757 | if (dst_pte)
|
| 1758 | pte_unmap(dst_pte);
|
| 1759 | mmu_notifier_invalidate_range_end(&range);
|
| 1760 | if (si)
|
| 1761 | put_swap_device(si);
|
| 1762 |
|
| 1763 | return ret;
|
| 1764 | }
|
| 1765 |
|
| 1766 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 1767 | static inline bool move_splits_huge_pmd(unsigned long dst_addr,
|
| 1768 | unsigned long src_addr,
|
| 1769 | unsigned long src_end)
|
| 1770 | {
|
| 1771 | return (src_addr & ~HPAGE_PMD_MASK) || (dst_addr & ~HPAGE_PMD_MASK) ||
|
| 1772 | src_end - src_addr < HPAGE_PMD_SIZE;
|
| 1773 | }
|
| 1774 | #else
|
| 1775 | static inline bool move_splits_huge_pmd(unsigned long dst_addr,
|
| 1776 | unsigned long src_addr,
|
| 1777 | unsigned long src_end)
|
| 1778 | {
|
| 1779 |
|
| 1780 | return false;
|
| 1781 | }
|
| 1782 | #endif
|
| 1783 |
|
| 1784 | static inline bool vma_move_compatible(struct vm_area_struct *vma)
|
| 1785 | {
|
| 1786 | return !(vma->vm_flags & (VM_PFNMAP | VM_IO | VM_HUGETLB |
|
| 1787 | VM_MIXEDMAP | VM_SHADOW_STACK));
|
| 1788 | }
|
| 1789 |
|
| 1790 | static int validate_move_areas(struct userfaultfd_ctx *ctx,
|
| 1791 | struct vm_area_struct *src_vma,
|
| 1792 | struct vm_area_struct *dst_vma)
|
| 1793 | {
|
| 1794 |
|
| 1795 | if ((src_vma->vm_flags & VM_ACCESS_FLAGS) != (dst_vma->vm_flags & VM_ACCESS_FLAGS) ||
|
| 1796 | pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot))
|
| 1797 | return -EINVAL;
|
| 1798 |
|
| 1799 |
|
| 1800 | if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED))
|
| 1801 | return -EINVAL;
|
| 1802 |
|
| 1803 | |
| 1804 | * For now, we keep it simple and only move between writable VMAs.
|
| 1805 | * Access flags are equal, therefore checking only the source is enough.
|
| 1806 |
|
| 1807 | if (!(src_vma->vm_flags & VM_WRITE))
|
| 1808 | return -EINVAL;
|
| 1809 |
|
| 1810 |
|
| 1811 | if (!vma_move_compatible(src_vma) || !vma_move_compatible(dst_vma))
|
| 1812 | return -EINVAL;
|
| 1813 |
|
| 1814 |
|
| 1815 | if (!dst_vma->vm_userfaultfd_ctx.ctx ||
|
| 1816 | dst_vma->vm_userfaultfd_ctx.ctx != ctx)
|
| 1817 | return -EINVAL;
|
| 1818 |
|
| 1819 |
|
| 1820 | if (!vma_is_anonymous(src_vma) || !vma_is_anonymous(dst_vma))
|
| 1821 | return -EINVAL;
|
| 1822 |
|
| 1823 | return 0;
|
| 1824 | }
|
| 1825 |
|
| 1826 | static __always_inline
|
| 1827 | int find_vmas_mm_locked(struct mm_struct *mm,
|
| 1828 | unsigned long dst_start,
|
| 1829 | unsigned long src_start,
|
| 1830 | struct vm_area_struct **dst_vmap,
|
| 1831 | struct vm_area_struct **src_vmap)
|
| 1832 | {
|
| 1833 | struct vm_area_struct *vma;
|
| 1834 |
|
| 1835 | mmap_assert_locked(mm);
|
| 1836 | vma = find_vma_and_prepare_anon(mm, dst_start);
|
| 1837 | if (IS_ERR(vma))
|
| 1838 | return PTR_ERR(vma);
|
| 1839 |
|
| 1840 | *dst_vmap = vma;
|
| 1841 |
|
| 1842 | if (src_start >= vma->vm_start && src_start < vma->vm_end)
|
| 1843 | goto out_success;
|
| 1844 |
|
| 1845 | vma = vma_lookup(mm, src_start);
|
| 1846 | if (!vma)
|
| 1847 | return -ENOENT;
|
| 1848 | out_success:
|
| 1849 | *src_vmap = vma;
|
| 1850 | return 0;
|
| 1851 | }
|
| 1852 |
|
| 1853 | #ifdef CONFIG_PER_VMA_LOCK
|
| 1854 | static int uffd_move_lock(struct mm_struct *mm,
|
| 1855 | unsigned long dst_start,
|
| 1856 | unsigned long src_start,
|
| 1857 | struct vm_area_struct **dst_vmap,
|
| 1858 | struct vm_area_struct **src_vmap)
|
| 1859 | {
|
| 1860 | struct vm_area_struct *vma;
|
| 1861 | int err;
|
| 1862 |
|
| 1863 | vma = uffd_lock_vma(mm, dst_start);
|
| 1864 | if (IS_ERR(vma))
|
| 1865 | return PTR_ERR(vma);
|
| 1866 |
|
| 1867 | *dst_vmap = vma;
|
| 1868 | |
| 1869 | * Skip finding src_vma if src_start is in dst_vma. This also ensures
|
| 1870 | * that we don't lock the same vma twice.
|
| 1871 |
|
| 1872 | if (src_start >= vma->vm_start && src_start < vma->vm_end) {
|
| 1873 | *src_vmap = vma;
|
| 1874 | return 0;
|
| 1875 | }
|
| 1876 |
|
| 1877 | |
| 1878 | * Using uffd_lock_vma() to get src_vma can lead to following deadlock:
|
| 1879 | *
|
| 1880 | * Thread1 Thread2
|
| 1881 | * ------- -------
|
| 1882 | * vma_start_read(dst_vma)
|
| 1883 | * mmap_write_lock(mm)
|
| 1884 | * vma_start_write(src_vma)
|
| 1885 | * vma_start_read(src_vma)
|
| 1886 | * mmap_read_lock(mm)
|
| 1887 | * vma_start_write(dst_vma)
|
| 1888 |
|
| 1889 | *src_vmap = lock_vma_under_rcu(mm, src_start);
|
| 1890 | if (likely(*src_vmap))
|
| 1891 | return 0;
|
| 1892 |
|
| 1893 |
|
| 1894 | vma_end_read(*dst_vmap);
|
| 1895 |
|
| 1896 | mmap_read_lock(mm);
|
| 1897 | err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap);
|
| 1898 | if (err)
|
| 1899 | goto out;
|
| 1900 |
|
| 1901 | if (!vma_start_read_locked(*dst_vmap)) {
|
| 1902 | err = -EAGAIN;
|
| 1903 | goto out;
|
| 1904 | }
|
| 1905 |
|
| 1906 |
|
| 1907 | if (*dst_vmap == *src_vmap)
|
| 1908 | goto out;
|
| 1909 |
|
| 1910 | if (!vma_start_read_locked_nested(*src_vmap, SINGLE_DEPTH_NESTING)) {
|
| 1911 |
|
| 1912 | vma_end_read(*dst_vmap);
|
| 1913 | err = -EAGAIN;
|
| 1914 | }
|
| 1915 | out:
|
| 1916 | mmap_read_unlock(mm);
|
| 1917 | return err;
|
| 1918 | }
|
| 1919 |
|
| 1920 | static void uffd_move_unlock(struct vm_area_struct *dst_vma,
|
| 1921 | struct vm_area_struct *src_vma)
|
| 1922 | {
|
| 1923 | vma_end_read(src_vma);
|
| 1924 | if (src_vma != dst_vma)
|
| 1925 | vma_end_read(dst_vma);
|
| 1926 | }
|
| 1927 |
|
| 1928 | #else
|
| 1929 |
|
| 1930 | static int uffd_move_lock(struct mm_struct *mm,
|
| 1931 | unsigned long dst_start,
|
| 1932 | unsigned long src_start,
|
| 1933 | struct vm_area_struct **dst_vmap,
|
| 1934 | struct vm_area_struct **src_vmap)
|
| 1935 | {
|
| 1936 | int err;
|
| 1937 |
|
| 1938 | mmap_read_lock(mm);
|
| 1939 | err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap);
|
| 1940 | if (err)
|
| 1941 | mmap_read_unlock(mm);
|
| 1942 | return err;
|
| 1943 | }
|
| 1944 |
|
| 1945 | static void uffd_move_unlock(struct vm_area_struct *dst_vma,
|
| 1946 | struct vm_area_struct *src_vma)
|
| 1947 | {
|
| 1948 | mmap_assert_locked(src_vma->vm_mm);
|
| 1949 | mmap_read_unlock(dst_vma->vm_mm);
|
| 1950 | }
|
| 1951 | #endif
|
| 1952 |
|
| 1953 | |
| 1954 | * move_pages - move arbitrary anonymous pages of an existing vma
|
| 1955 | * @ctx: pointer to the userfaultfd context
|
| 1956 | * @dst_start: start of the destination virtual memory range
|
| 1957 | * @src_start: start of the source virtual memory range
|
| 1958 | * @len: length of the virtual memory range
|
| 1959 | * @mode: flags from uffdio_move.mode
|
| 1960 | *
|
| 1961 | * It will either use the mmap_lock in read mode or per-vma locks
|
| 1962 | *
|
| 1963 | * move_pages() remaps arbitrary anonymous pages atomically in zero
|
| 1964 | * copy. It only works on non shared anonymous pages because those can
|
| 1965 | * be relocated without generating non linear anon_vmas in the rmap
|
| 1966 | * code.
|
| 1967 | *
|
| 1968 | * It provides a zero copy mechanism to handle userspace page faults.
|
| 1969 | * The source vma pages should have mapcount == 1, which can be
|
| 1970 | * enforced by using madvise(MADV_DONTFORK) on src vma.
|
| 1971 | *
|
| 1972 | * The thread receiving the page during the userland page fault
|
| 1973 | * will receive the faulting page in the source vma through the network,
|
| 1974 | * storage or any other I/O device (MADV_DONTFORK in the source vma
|
| 1975 | * avoids move_pages() to fail with -EBUSY if the process forks before
|
| 1976 | * move_pages() is called), then it will call move_pages() to map the
|
| 1977 | * page in the faulting address in the destination vma.
|
| 1978 | *
|
| 1979 | * This userfaultfd command works purely via pagetables, so it's the
|
| 1980 | * most efficient way to move physical non shared anonymous pages
|
| 1981 | * across different virtual addresses. Unlike mremap()/mmap()/munmap()
|
| 1982 | * it does not create any new vmas. The mapping in the destination
|
| 1983 | * address is atomic.
|
| 1984 | *
|
| 1985 | * It only works if the vma protection bits are identical from the
|
| 1986 | * source and destination vma.
|
| 1987 | *
|
| 1988 | * It can remap non shared anonymous pages within the same vma too.
|
| 1989 | *
|
| 1990 | * If the source virtual memory range has any unmapped holes, or if
|
| 1991 | * the destination virtual memory range is not a whole unmapped hole,
|
| 1992 | * move_pages() will fail respectively with -ENOENT or -EEXIST. This
|
| 1993 | * provides a very strict behavior to avoid any chance of memory
|
| 1994 | * corruption going unnoticed if there are userland race conditions.
|
| 1995 | * Only one thread should resolve the userland page fault at any given
|
| 1996 | * time for any given faulting address. This means that if two threads
|
| 1997 | * try to both call move_pages() on the same destination address at the
|
| 1998 | * same time, the second thread will get an explicit error from this
|
| 1999 | * command.
|
| 2000 | *
|
| 2001 | * The command retval will return "len" is successful. The command
|
| 2002 | * however can be interrupted by fatal signals or errors. If
|
| 2003 | * interrupted it will return the number of bytes successfully
|
| 2004 | * remapped before the interruption if any, or the negative error if
|
| 2005 | * none. It will never return zero. Either it will return an error or
|
| 2006 | * an amount of bytes successfully moved. If the retval reports a
|
| 2007 | * "short" remap, the move_pages() command should be repeated by
|
| 2008 | * userland with src+retval, dst+reval, len-retval if it wants to know
|
| 2009 | * about the error that interrupted it.
|
| 2010 | *
|
| 2011 | * The UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES flag can be specified to
|
| 2012 | * prevent -ENOENT errors to materialize if there are holes in the
|
| 2013 | * source virtual range that is being remapped. The holes will be
|
| 2014 | * accounted as successfully remapped in the retval of the
|
| 2015 | * command. This is mostly useful to remap hugepage naturally aligned
|
| 2016 | * virtual regions without knowing if there are transparent hugepage
|
| 2017 | * in the regions or not, but preventing the risk of having to split
|
| 2018 | * the hugepmd during the remap.
|
| 2019 |
|
| 2020 | static ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start,
|
| 2021 | unsigned long src_start, unsigned long len, __u64 mode)
|
| 2022 | {
|
| 2023 | struct mm_struct *mm = ctx->mm;
|
| 2024 | struct vm_area_struct *src_vma, *dst_vma;
|
| 2025 | unsigned long src_addr, dst_addr, src_end;
|
| 2026 | pmd_t *src_pmd, *dst_pmd;
|
| 2027 | long err = -EINVAL;
|
| 2028 | ssize_t moved = 0;
|
| 2029 |
|
| 2030 |
|
| 2031 | VM_WARN_ON_ONCE(src_start & ~PAGE_MASK);
|
| 2032 | VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK);
|
| 2033 | VM_WARN_ON_ONCE(len & ~PAGE_MASK);
|
| 2034 |
|
| 2035 |
|
| 2036 | VM_WARN_ON_ONCE(src_start + len < src_start);
|
| 2037 | VM_WARN_ON_ONCE(dst_start + len < dst_start);
|
| 2038 |
|
| 2039 | err = uffd_move_lock(mm, dst_start, src_start, &dst_vma, &src_vma);
|
| 2040 | if (err)
|
| 2041 | goto out;
|
| 2042 |
|
| 2043 |
|
| 2044 | err = -EAGAIN;
|
| 2045 | down_read(&ctx->map_changing_lock);
|
| 2046 | if (likely(atomic_read(&ctx->mmap_changing)))
|
| 2047 | goto out_unlock;
|
| 2048 | |
| 2049 | * Make sure the vma is not shared, that the src and dst remap
|
| 2050 | * ranges are both valid and fully within a single existing
|
| 2051 | * vma.
|
| 2052 |
|
| 2053 | err = -EINVAL;
|
| 2054 | if (src_vma->vm_flags & VM_SHARED)
|
| 2055 | goto out_unlock;
|
| 2056 | if (src_start + len > src_vma->vm_end)
|
| 2057 | goto out_unlock;
|
| 2058 |
|
| 2059 | if (dst_vma->vm_flags & VM_SHARED)
|
| 2060 | goto out_unlock;
|
| 2061 | if (dst_start + len > dst_vma->vm_end)
|
| 2062 | goto out_unlock;
|
| 2063 |
|
| 2064 | err = validate_move_areas(ctx, src_vma, dst_vma);
|
| 2065 | if (err)
|
| 2066 | goto out_unlock;
|
| 2067 |
|
| 2068 | for (src_addr = src_start, dst_addr = dst_start, src_end = src_start + len;
|
| 2069 | src_addr < src_end;) {
|
| 2070 | spinlock_t *ptl;
|
| 2071 | pmd_t dst_pmdval;
|
| 2072 | unsigned long step_size;
|
| 2073 |
|
| 2074 | |
| 2075 | * Below works because anonymous area would not have a
|
| 2076 | * transparent huge PUD. If file-backed support is added,
|
| 2077 | * that case would need to be handled here.
|
| 2078 |
|
| 2079 | src_pmd = mm_find_pmd(mm, src_addr);
|
| 2080 | if (unlikely(!src_pmd)) {
|
| 2081 | if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) {
|
| 2082 | err = -ENOENT;
|
| 2083 | break;
|
| 2084 | }
|
| 2085 | src_pmd = mm_alloc_pmd(mm, src_addr);
|
| 2086 | if (unlikely(!src_pmd)) {
|
| 2087 | err = -ENOMEM;
|
| 2088 | break;
|
| 2089 | }
|
| 2090 | }
|
| 2091 | dst_pmd = mm_alloc_pmd(mm, dst_addr);
|
| 2092 | if (unlikely(!dst_pmd)) {
|
| 2093 | err = -ENOMEM;
|
| 2094 | break;
|
| 2095 | }
|
| 2096 |
|
| 2097 | dst_pmdval = pmdp_get_lockless(dst_pmd);
|
| 2098 | |
| 2099 | * If the dst_pmd is mapped as THP don't override it and just
|
| 2100 | * be strict. If dst_pmd changes into TPH after this check, the
|
| 2101 | * move_pages_huge_pmd() will detect the change and retry
|
| 2102 | * while move_pages_pte() will detect the change and fail.
|
| 2103 |
|
| 2104 | if (unlikely(pmd_trans_huge(dst_pmdval))) {
|
| 2105 | err = -EEXIST;
|
| 2106 | break;
|
| 2107 | }
|
| 2108 |
|
| 2109 | ptl = pmd_trans_huge_lock(src_pmd, src_vma);
|
| 2110 | if (ptl) {
|
| 2111 |
|
| 2112 | if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) ||
|
| 2113 | !pmd_none(dst_pmdval)) {
|
| 2114 |
|
| 2115 | if (pmd_present(*src_pmd)) {
|
| 2116 | struct folio *folio = pmd_folio(*src_pmd);
|
| 2117 |
|
| 2118 | if (!is_huge_zero_folio(folio) &&
|
| 2119 | !PageAnonExclusive(&folio->page)) {
|
| 2120 | spin_unlock(ptl);
|
| 2121 | err = -EBUSY;
|
| 2122 | break;
|
| 2123 | }
|
| 2124 | }
|
| 2125 |
|
| 2126 | spin_unlock(ptl);
|
| 2127 | split_huge_pmd(src_vma, src_pmd, src_addr);
|
| 2128 |
|
| 2129 | continue;
|
| 2130 | }
|
| 2131 |
|
| 2132 | err = move_pages_huge_pmd(mm, dst_pmd, src_pmd,
|
| 2133 | dst_pmdval, dst_vma, src_vma,
|
| 2134 | dst_addr, src_addr);
|
| 2135 | step_size = HPAGE_PMD_SIZE;
|
| 2136 | } else {
|
| 2137 | long ret;
|
| 2138 |
|
| 2139 | if (pmd_none(*src_pmd)) {
|
| 2140 | if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) {
|
| 2141 | err = -ENOENT;
|
| 2142 | break;
|
| 2143 | }
|
| 2144 | if (unlikely(__pte_alloc(mm, src_pmd))) {
|
| 2145 | err = -ENOMEM;
|
| 2146 | break;
|
| 2147 | }
|
| 2148 | }
|
| 2149 |
|
| 2150 | if (unlikely(pte_alloc(mm, dst_pmd))) {
|
| 2151 | err = -ENOMEM;
|
| 2152 | break;
|
| 2153 | }
|
| 2154 |
|
| 2155 | ret = move_pages_ptes(mm, dst_pmd, src_pmd,
|
| 2156 | dst_vma, src_vma, dst_addr,
|
| 2157 | src_addr, src_end - src_addr, mode);
|
| 2158 | if (ret < 0)
|
| 2159 | err = ret;
|
| 2160 | else
|
| 2161 | step_size = ret;
|
| 2162 | }
|
| 2163 |
|
| 2164 | cond_resched();
|
| 2165 |
|
| 2166 | if (fatal_signal_pending(current)) {
|
| 2167 |
|
| 2168 | if (!err || err == -EAGAIN)
|
| 2169 | err = -EINTR;
|
| 2170 | break;
|
| 2171 | }
|
| 2172 |
|
| 2173 | if (err) {
|
| 2174 | if (err == -EAGAIN) {
|
| 2175 | err = 0;
|
| 2176 | continue;
|
| 2177 | }
|
| 2178 | break;
|
| 2179 | }
|
| 2180 |
|
| 2181 |
|
| 2182 | dst_addr += step_size;
|
| 2183 | src_addr += step_size;
|
| 2184 | moved += step_size;
|
| 2185 | }
|
| 2186 |
|
| 2187 | out_unlock:
|
| 2188 | up_read(&ctx->map_changing_lock);
|
| 2189 | uffd_move_unlock(dst_vma, src_vma);
|
| 2190 | out:
|
| 2191 | VM_WARN_ON_ONCE(moved < 0);
|
| 2192 | VM_WARN_ON_ONCE(err > 0);
|
| 2193 | VM_WARN_ON_ONCE(!moved && !err);
|
| 2194 | return moved ? moved : err;
|
| 2195 | }
|
| 2196 |
|
| 2197 | static bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags,
|
| 2198 | bool wp_async)
|
| 2199 | {
|
| 2200 | const struct vm_uffd_ops *ops = vma_uffd_ops(vma);
|
| 2201 |
|
| 2202 | if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK))
|
| 2203 | return false;
|
| 2204 |
|
| 2205 | if (!is_vm_hugetlb_page(vma) && (vma->vm_flags & VM_SPECIAL))
|
| 2206 | return false;
|
| 2207 |
|
| 2208 | vm_flags &= __VM_UFFD_FLAGS;
|
| 2209 |
|
| 2210 | |
| 2211 | * If WP is the only mode enabled and context is wp async, allow any
|
| 2212 | * memory type.
|
| 2213 |
|
| 2214 | if (wp_async && (vm_flags == VM_UFFD_WP))
|
| 2215 | return true;
|
| 2216 |
|
| 2217 |
|
| 2218 | if (!ops)
|
| 2219 | return false;
|
| 2220 |
|
| 2221 | |
| 2222 | * If user requested uffd-wp but not enabled pte markers for
|
| 2223 | * uffd-wp, then only anonymous memory is supported
|
| 2224 |
|
| 2225 | if (!uffd_supports_wp_marker() && (vm_flags & VM_UFFD_WP) &&
|
| 2226 | !vma_is_anonymous(vma))
|
| 2227 | return false;
|
| 2228 |
|
| 2229 | return ops->can_userfault(vma, vm_flags);
|
| 2230 | }
|
| 2231 |
|
| 2232 | static void userfaultfd_set_vm_flags(struct vm_area_struct *vma,
|
| 2233 | vm_flags_t vm_flags)
|
| 2234 | {
|
| 2235 | const bool uffd_wp_changed = (vma->vm_flags ^ vm_flags) & VM_UFFD_WP;
|
| 2236 |
|
| 2237 | vm_flags_reset(vma, vm_flags);
|
| 2238 | |
| 2239 | * For shared mappings, we want to enable writenotify while
|
| 2240 | * userfaultfd-wp is enabled (see vma_wants_writenotify()). We'll simply
|
| 2241 | * recalculate vma->vm_page_prot whenever userfaultfd-wp changes.
|
| 2242 |
|
| 2243 | if ((vma->vm_flags & VM_SHARED) && uffd_wp_changed)
|
| 2244 | vma_set_page_prot(vma);
|
| 2245 | }
|
| 2246 |
|
| 2247 | static void userfaultfd_set_ctx(struct vm_area_struct *vma,
|
| 2248 | struct userfaultfd_ctx *ctx,
|
| 2249 | vm_flags_t vm_flags)
|
| 2250 | {
|
| 2251 | vma_start_write(vma);
|
| 2252 | vma->vm_userfaultfd_ctx = (struct vm_userfaultfd_ctx){ctx};
|
| 2253 | userfaultfd_set_vm_flags(vma,
|
| 2254 | (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags);
|
| 2255 | }
|
| 2256 |
|
| 2257 | static void userfaultfd_reset_ctx(struct vm_area_struct *vma)
|
| 2258 | {
|
| 2259 | userfaultfd_set_ctx(vma, NULL, 0);
|
| 2260 | }
|
| 2261 |
|
| 2262 | static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
|
| 2263 | struct vm_area_struct *prev,
|
| 2264 | struct vm_area_struct *vma,
|
| 2265 | unsigned long start,
|
| 2266 | unsigned long end)
|
| 2267 | {
|
| 2268 | struct vm_area_struct *ret;
|
| 2269 | bool give_up_on_oom = false;
|
| 2270 | vma_flags_t new_vma_flags = vma->flags;
|
| 2271 |
|
| 2272 | vma_flags_clear_mask(&new_vma_flags, __VMA_UFFD_FLAGS);
|
| 2273 |
|
| 2274 | |
| 2275 | * If we are modifying only and not splitting, just give up on the merge
|
| 2276 | * if OOM prevents us from merging successfully.
|
| 2277 |
|
| 2278 | if (start == vma->vm_start && end == vma->vm_end)
|
| 2279 | give_up_on_oom = true;
|
| 2280 |
|
| 2281 |
|
| 2282 | if (userfaultfd_protected(vma)) {
|
| 2283 | unsigned int mm_cp_flags = 0;
|
| 2284 | struct mmu_gather tlb;
|
| 2285 |
|
| 2286 | if (userfaultfd_wp(vma))
|
| 2287 | mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
|
| 2288 | if (userfaultfd_rwp(vma))
|
| 2289 | mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
|
| 2290 | if (vma_wants_manual_pte_write_upgrade(vma))
|
| 2291 | mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
|
| 2292 |
|
| 2293 | tlb_gather_mmu(&tlb, vma->vm_mm);
|
| 2294 | change_protection(&tlb, vma, start, end, mm_cp_flags);
|
| 2295 | tlb_finish_mmu(&tlb);
|
| 2296 | }
|
| 2297 |
|
| 2298 | ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
|
| 2299 | &new_vma_flags, NULL_VM_UFFD_CTX,
|
| 2300 | give_up_on_oom);
|
| 2301 |
|
| 2302 | |
| 2303 | * In the vma_merge() successful mprotect-like case 8:
|
| 2304 | * the next vma was merged into the current one and
|
| 2305 | * the current one has not been updated yet.
|
| 2306 |
|
| 2307 | if (!IS_ERR(ret))
|
| 2308 | userfaultfd_reset_ctx(ret);
|
| 2309 |
|
| 2310 | return ret;
|
| 2311 | }
|
| 2312 |
|
| 2313 |
|
| 2314 | static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
|
| 2315 | struct vm_area_struct *vma,
|
| 2316 | vm_flags_t vm_flags,
|
| 2317 | unsigned long start, unsigned long end,
|
| 2318 | bool wp_async)
|
| 2319 | {
|
| 2320 | vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags);
|
| 2321 | VMA_ITERATOR(vmi, ctx->mm, start);
|
| 2322 | struct vm_area_struct *prev = vma_prev(&vmi);
|
| 2323 | unsigned long vma_end;
|
| 2324 | vma_flags_t new_vma_flags;
|
| 2325 |
|
| 2326 | if (vma->vm_start < start)
|
| 2327 | prev = vma;
|
| 2328 |
|
| 2329 | for_each_vma_range(vmi, vma, end) {
|
| 2330 | cond_resched();
|
| 2331 |
|
| 2332 | VM_WARN_ON_ONCE(!vma_can_userfault(vma, vm_flags, wp_async));
|
| 2333 | VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx &&
|
| 2334 | vma->vm_userfaultfd_ctx.ctx != ctx);
|
| 2335 | VM_WARN_ON_ONCE(!vma_test(vma, VMA_MAYWRITE_BIT));
|
| 2336 |
|
| 2337 | |
| 2338 | * Nothing to do: this vma is already registered into this
|
| 2339 | * userfaultfd and with the right tracking mode too.
|
| 2340 |
|
| 2341 | if (vma->vm_userfaultfd_ctx.ctx == ctx &&
|
| 2342 | vma_test_all_mask(vma, vma_flags))
|
| 2343 | goto skip;
|
| 2344 |
|
| 2345 | |
| 2346 | * Pre-scan in userfaultfd_register() already rejected mode
|
| 2347 | * switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a
|
| 2348 | * stray bit here is a bug.
|
| 2349 |
|
| 2350 | VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx &&
|
| 2351 | vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags);
|
| 2352 |
|
| 2353 | if (vma->vm_start > start)
|
| 2354 | start = vma->vm_start;
|
| 2355 | vma_end = min(end, vma->vm_end);
|
| 2356 |
|
| 2357 | new_vma_flags = vma->flags;
|
| 2358 | vma_flags_clear_mask(&new_vma_flags, __VMA_UFFD_FLAGS);
|
| 2359 | vma_flags_set_mask(&new_vma_flags, vma_flags);
|
| 2360 |
|
| 2361 | vma = vma_modify_flags_uffd(&vmi, prev, vma, start, vma_end,
|
| 2362 | &new_vma_flags,
|
| 2363 | (struct vm_userfaultfd_ctx){ctx},
|
| 2364 | false);
|
| 2365 | if (IS_ERR(vma))
|
| 2366 | return PTR_ERR(vma);
|
| 2367 |
|
| 2368 | |
| 2369 | * In the vma_merge() successful mprotect-like case 8:
|
| 2370 | * the next vma was merged into the current one and
|
| 2371 | * the current one has not been updated yet.
|
| 2372 |
|
| 2373 | userfaultfd_set_ctx(vma, ctx, vm_flags);
|
| 2374 |
|
| 2375 | if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
|
| 2376 | hugetlb_unshare_all_pmds(vma);
|
| 2377 |
|
| 2378 | skip:
|
| 2379 | prev = vma;
|
| 2380 | start = vma->vm_end;
|
| 2381 | }
|
| 2382 |
|
| 2383 | return 0;
|
| 2384 | }
|
| 2385 |
|
| 2386 | static void userfaultfd_release_new(struct userfaultfd_ctx *ctx)
|
| 2387 | {
|
| 2388 | struct mm_struct *mm = ctx->mm;
|
| 2389 | struct vm_area_struct *vma;
|
| 2390 | VMA_ITERATOR(vmi, mm, 0);
|
| 2391 |
|
| 2392 |
|
| 2393 | mmap_write_lock(mm);
|
| 2394 | for_each_vma(vmi, vma) {
|
| 2395 | if (vma->vm_userfaultfd_ctx.ctx == ctx)
|
| 2396 | userfaultfd_reset_ctx(vma);
|
| 2397 | }
|
| 2398 | mmap_write_unlock(mm);
|
| 2399 | }
|
| 2400 |
|
| 2401 | static void userfaultfd_release_all(struct mm_struct *mm,
|
| 2402 | struct userfaultfd_ctx *ctx)
|
| 2403 | {
|
| 2404 | struct vm_area_struct *vma, *prev;
|
| 2405 | VMA_ITERATOR(vmi, mm, 0);
|
| 2406 |
|
| 2407 | if (!mmget_not_zero(mm))
|
| 2408 | return;
|
| 2409 |
|
| 2410 | |
| 2411 | * Flush page faults out of all CPUs. NOTE: all page faults
|
| 2412 | * must be retried without returning VM_FAULT_SIGBUS if
|
| 2413 | * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
|
| 2414 | * changes while handle_userfault released the mmap_lock. So
|
| 2415 | * it's critical that released is set to true (above), before
|
| 2416 | * taking the mmap_lock for writing.
|
| 2417 |
|
| 2418 | mmap_write_lock(mm);
|
| 2419 | prev = NULL;
|
| 2420 | for_each_vma(vmi, vma) {
|
| 2421 | cond_resched();
|
| 2422 | VM_WARN_ON_ONCE(!!vma->vm_userfaultfd_ctx.ctx ^
|
| 2423 | !!(vma->vm_flags & __VM_UFFD_FLAGS));
|
| 2424 | if (vma->vm_userfaultfd_ctx.ctx != ctx) {
|
| 2425 | prev = vma;
|
| 2426 | continue;
|
| 2427 | }
|
| 2428 |
|
| 2429 | vma = userfaultfd_clear_vma(&vmi, prev, vma,
|
| 2430 | vma->vm_start, vma->vm_end);
|
| 2431 | prev = vma;
|
| 2432 | }
|
| 2433 | mmap_write_unlock(mm);
|
| 2434 | mmput(mm);
|
| 2435 | }
|
| 2436 |
|
| 2437 | static int sysctl_unprivileged_userfaultfd __read_mostly;
|
| 2438 |
|
| 2439 | #ifdef CONFIG_SYSCTL
|
| 2440 | static const struct ctl_table vm_userfaultfd_table[] = {
|
| 2441 | {
|
| 2442 | .procname = "unprivileged_userfaultfd",
|
| 2443 | .data = &sysctl_unprivileged_userfaultfd,
|
| 2444 | .maxlen = sizeof(sysctl_unprivileged_userfaultfd),
|
| 2445 | .mode = 0644,
|
| 2446 | .proc_handler = proc_dointvec_minmax,
|
| 2447 | .extra1 = SYSCTL_ZERO,
|
| 2448 | .extra2 = SYSCTL_ONE,
|
| 2449 | },
|
| 2450 | };
|
| 2451 | #endif
|
| 2452 |
|
| 2453 | static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init;
|
| 2454 |
|
| 2455 | struct userfaultfd_fork_ctx {
|
| 2456 | struct userfaultfd_ctx *orig;
|
| 2457 | struct userfaultfd_ctx *new;
|
| 2458 | struct list_head list;
|
| 2459 | };
|
| 2460 |
|
| 2461 | struct userfaultfd_unmap_ctx {
|
| 2462 | struct userfaultfd_ctx *ctx;
|
| 2463 | unsigned long start;
|
| 2464 | unsigned long end;
|
| 2465 | struct list_head list;
|
| 2466 | };
|
| 2467 |
|
| 2468 | struct userfaultfd_wait_queue {
|
| 2469 | struct uffd_msg msg;
|
| 2470 | wait_queue_entry_t wq;
|
| 2471 | struct userfaultfd_ctx *ctx;
|
| 2472 | bool waken;
|
| 2473 | };
|
| 2474 |
|
| 2475 | struct userfaultfd_wake_range {
|
| 2476 | unsigned long start;
|
| 2477 | unsigned long len;
|
| 2478 | };
|
| 2479 |
|
| 2480 |
|
| 2481 | #define UFFD_FEATURE_INITIALIZED (1u << 31)
|
| 2482 |
|
| 2483 | |
| 2484 | * UFFDIO_SET_MODE updates ctx->features under mmap_write_lock with
|
| 2485 | * WRITE_ONCE; readers that run outside mmap_read_lock or the per-VMA
|
| 2486 | * lock (poll/read_iter/ioctl, fdinfo) must pair with READ_ONCE.
|
| 2487 |
|
| 2488 | static unsigned int userfaultfd_features(struct userfaultfd_ctx *ctx)
|
| 2489 | {
|
| 2490 | return READ_ONCE(ctx->features);
|
| 2491 | }
|
| 2492 |
|
| 2493 | static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
|
| 2494 | {
|
| 2495 | return userfaultfd_features(ctx) & UFFD_FEATURE_INITIALIZED;
|
| 2496 | }
|
| 2497 |
|
| 2498 | static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx)
|
| 2499 | {
|
| 2500 | return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_WP_ASYNC);
|
| 2501 | }
|
| 2502 |
|
| 2503 | static bool userfaultfd_rwp_async_ctx(struct userfaultfd_ctx *ctx)
|
| 2504 | {
|
| 2505 | return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_RWP_ASYNC);
|
| 2506 | }
|
| 2507 |
|
| 2508 | |
| 2509 | * Whether WP_UNPOPULATED is enabled on the uffd context. It is only
|
| 2510 | * meaningful when userfaultfd_wp()==true on the vma and when it's
|
| 2511 | * anonymous.
|
| 2512 |
|
| 2513 | bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma)
|
| 2514 | {
|
| 2515 | struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
|
| 2516 |
|
| 2517 | if (!ctx)
|
| 2518 | return false;
|
| 2519 |
|
| 2520 | return userfaultfd_features(ctx) & UFFD_FEATURE_WP_UNPOPULATED;
|
| 2521 | }
|
| 2522 |
|
| 2523 | static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
|
| 2524 | int wake_flags, void *key)
|
| 2525 | {
|
| 2526 | struct userfaultfd_wake_range *range = key;
|
| 2527 | int ret;
|
| 2528 | struct userfaultfd_wait_queue *uwq;
|
| 2529 | unsigned long start, len;
|
| 2530 |
|
| 2531 | uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
|
| 2532 | ret = 0;
|
| 2533 |
|
| 2534 | start = range->start;
|
| 2535 | len = range->len;
|
| 2536 | if (len && (start > uwq->msg.arg.pagefault.address ||
|
| 2537 | start + len <= uwq->msg.arg.pagefault.address))
|
| 2538 | goto out;
|
| 2539 | WRITE_ONCE(uwq->waken, true);
|
| 2540 | |
| 2541 | * The Program-Order guarantees provided by the scheduler
|
| 2542 | * ensure uwq->waken is visible before the task is woken.
|
| 2543 |
|
| 2544 | ret = wake_up_state(wq->private, mode);
|
| 2545 | if (ret) {
|
| 2546 | |
| 2547 | * Wake only once, autoremove behavior.
|
| 2548 | *
|
| 2549 | * After the effect of list_del_init is visible to the other
|
| 2550 | * CPUs, the waitqueue may disappear from under us, see the
|
| 2551 | * !list_empty_careful() in handle_userfault().
|
| 2552 | *
|
| 2553 | * try_to_wake_up() has an implicit smp_mb(), and the
|
| 2554 | * wq->private is read before calling the extern function
|
| 2555 | * "wake_up_state" (which in turns calls try_to_wake_up).
|
| 2556 |
|
| 2557 | list_del_init(&wq->entry);
|
| 2558 | }
|
| 2559 | out:
|
| 2560 | return ret;
|
| 2561 | }
|
| 2562 |
|
| 2563 | |
| 2564 | * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
|
| 2565 | * context.
|
| 2566 | * @ctx: [in] Pointer to the userfaultfd context.
|
| 2567 |
|
| 2568 | static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
|
| 2569 | {
|
| 2570 | refcount_inc(&ctx->refcount);
|
| 2571 | }
|
| 2572 |
|
| 2573 | |
| 2574 | * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
|
| 2575 | * context.
|
| 2576 | * @ctx: [in] Pointer to userfaultfd context.
|
| 2577 | *
|
| 2578 | * The userfaultfd context reference must have been previously acquired either
|
| 2579 | * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
|
| 2580 |
|
| 2581 | static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
|
| 2582 | {
|
| 2583 | if (refcount_dec_and_test(&ctx->refcount)) {
|
| 2584 | VM_WARN_ON_ONCE(spin_is_locked(&ctx->fault_pending_wqh.lock));
|
| 2585 | VM_WARN_ON_ONCE(waitqueue_active(&ctx->fault_pending_wqh));
|
| 2586 | VM_WARN_ON_ONCE(spin_is_locked(&ctx->fault_wqh.lock));
|
| 2587 | VM_WARN_ON_ONCE(waitqueue_active(&ctx->fault_wqh));
|
| 2588 | VM_WARN_ON_ONCE(spin_is_locked(&ctx->event_wqh.lock));
|
| 2589 | VM_WARN_ON_ONCE(waitqueue_active(&ctx->event_wqh));
|
| 2590 | VM_WARN_ON_ONCE(spin_is_locked(&ctx->fd_wqh.lock));
|
| 2591 | VM_WARN_ON_ONCE(waitqueue_active(&ctx->fd_wqh));
|
| 2592 | mmdrop(ctx->mm);
|
| 2593 | kmem_cache_free(userfaultfd_ctx_cachep, ctx);
|
| 2594 | }
|
| 2595 | }
|
| 2596 |
|
| 2597 | static inline void msg_init(struct uffd_msg *msg)
|
| 2598 | {
|
| 2599 | BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
|
| 2600 | |
| 2601 | * Must use memset to zero out the paddings or kernel data is
|
| 2602 | * leaked to userland.
|
| 2603 |
|
| 2604 | memset(msg, 0, sizeof(struct uffd_msg));
|
| 2605 | }
|
| 2606 |
|
| 2607 | static inline struct uffd_msg userfault_msg(unsigned long address,
|
| 2608 | unsigned long real_address,
|
| 2609 | unsigned int flags,
|
| 2610 | unsigned long reason,
|
| 2611 | unsigned int features)
|
| 2612 | {
|
| 2613 | struct uffd_msg msg;
|
| 2614 |
|
| 2615 | msg_init(&msg);
|
| 2616 | msg.event = UFFD_EVENT_PAGEFAULT;
|
| 2617 |
|
| 2618 | msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ?
|
| 2619 | real_address : address;
|
| 2620 |
|
| 2621 | |
| 2622 | * These flags indicate why the userfault occurred:
|
| 2623 | * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
|
| 2624 | * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
|
| 2625 | * - Neither of these flags being set indicates a MISSING fault.
|
| 2626 | *
|
| 2627 | * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
|
| 2628 | * fault. Otherwise, it was a read fault.
|
| 2629 |
|
| 2630 | if (flags & FAULT_FLAG_WRITE)
|
| 2631 | msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
|
| 2632 | if (reason & VM_UFFD_WP)
|
| 2633 | msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
|
| 2634 | if (reason & VM_UFFD_RWP)
|
| 2635 | msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP;
|
| 2636 | if (reason & VM_UFFD_MINOR)
|
| 2637 | msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
|
| 2638 | if (features & UFFD_FEATURE_THREAD_ID)
|
| 2639 | msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
|
| 2640 | return msg;
|
| 2641 | }
|
| 2642 |
|
| 2643 | #ifdef CONFIG_HUGETLB_PAGE
|
| 2644 | |
| 2645 | * Same functionality as userfaultfd_must_wait below with modifications for
|
| 2646 | * hugepmd ranges.
|
| 2647 |
|
| 2648 | static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
|
| 2649 | struct vm_fault *vmf,
|
| 2650 | unsigned long reason)
|
| 2651 | {
|
| 2652 | struct vm_area_struct *vma = vmf->vma;
|
| 2653 | pte_t *ptep, pte;
|
| 2654 |
|
| 2655 | assert_fault_locked(vmf);
|
| 2656 |
|
| 2657 | ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma));
|
| 2658 | if (!ptep)
|
| 2659 | return true;
|
| 2660 |
|
| 2661 | pte = huge_ptep_get(vma->vm_mm, vmf->address, ptep);
|
| 2662 |
|
| 2663 | |
| 2664 | * Lockless access: we're in a wait_event so it's ok if it
|
| 2665 | * changes under us.
|
| 2666 |
|
| 2667 |
|
| 2668 |
|
| 2669 | if (huge_pte_none(pte))
|
| 2670 | return true;
|
| 2671 |
|
| 2672 | if (pte_is_uffd_marker(pte))
|
| 2673 | return true;
|
| 2674 | |
| 2675 | * Concurrent migration may have replaced the present PTE with a
|
| 2676 | * non-marker swap entry between fault delivery and this lockless
|
| 2677 | * re-check. huge_pte_write() on a swap entry decodes random offset
|
| 2678 | * bits, so gate it on pte_present(). The migration completion path
|
| 2679 | * will re-deliver the fault if it still needs userspace.
|
| 2680 |
|
| 2681 | if (!pte_present(pte))
|
| 2682 | return false;
|
| 2683 | |
| 2684 | * If VMA has UFFD WP faults enabled and WP fault, wait for userspace to
|
| 2685 | * resolve the fault.
|
| 2686 |
|
| 2687 | if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
|
| 2688 | return true;
|
| 2689 | |
| 2690 | * PTE is still RW-protected (protnone with uffd bit), wait for
|
| 2691 | * resolution. Plain PROT_NONE without the marker is not an RWP fault.
|
| 2692 |
|
| 2693 | if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP))
|
| 2694 | return true;
|
| 2695 |
|
| 2696 | return false;
|
| 2697 | }
|
| 2698 | #else
|
| 2699 | static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
|
| 2700 | struct vm_fault *vmf,
|
| 2701 | unsigned long reason)
|
| 2702 | {
|
| 2703 |
|
| 2704 | VM_WARN_ON_ONCE(1);
|
| 2705 | return false;
|
| 2706 | }
|
| 2707 | #endif
|
| 2708 |
|
| 2709 | |
| 2710 | * Verify the pagetables are still not ok after having registered into
|
| 2711 | * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
|
| 2712 | * userfault that has already been resolved, if userfaultfd_read_iter and
|
| 2713 | * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
|
| 2714 | * threads.
|
| 2715 |
|
| 2716 | static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
|
| 2717 | struct vm_fault *vmf,
|
| 2718 | unsigned long reason)
|
| 2719 | {
|
| 2720 | struct mm_struct *mm = ctx->mm;
|
| 2721 | unsigned long address = vmf->address;
|
| 2722 | pgd_t *pgd;
|
| 2723 | p4d_t *p4d;
|
| 2724 | pud_t *pud;
|
| 2725 | pmd_t *pmd, _pmd;
|
| 2726 | pte_t *pte;
|
| 2727 | pte_t ptent;
|
| 2728 | bool ret;
|
| 2729 |
|
| 2730 | assert_fault_locked(vmf);
|
| 2731 |
|
| 2732 | pgd = pgd_offset(mm, address);
|
| 2733 | if (!pgd_present(*pgd))
|
| 2734 | return true;
|
| 2735 | p4d = p4d_offset(pgd, address);
|
| 2736 | if (!p4d_present(*p4d))
|
| 2737 | return true;
|
| 2738 | pud = pud_offset(p4d, address);
|
| 2739 | if (!pud_present(*pud))
|
| 2740 | return true;
|
| 2741 | pmd = pmd_offset(pud, address);
|
| 2742 | again:
|
| 2743 | _pmd = pmdp_get_lockless(pmd);
|
| 2744 | if (pmd_none(_pmd))
|
| 2745 | return true;
|
| 2746 |
|
| 2747 | |
| 2748 | * A race could arise which would result in a softleaf entry such as
|
| 2749 | * migration entry unexpectedly being present in the PMD, so explicitly
|
| 2750 | * check for this and bail out if so.
|
| 2751 |
|
| 2752 | if (!pmd_present(_pmd))
|
| 2753 | return false;
|
| 2754 |
|
| 2755 | if (pmd_trans_huge(_pmd)) {
|
| 2756 | if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
|
| 2757 | return true;
|
| 2758 | if (pmd_protnone(_pmd) && pmd_uffd(_pmd) &&
|
| 2759 | (reason & VM_UFFD_RWP))
|
| 2760 | return true;
|
| 2761 | return false;
|
| 2762 | }
|
| 2763 |
|
| 2764 | pte = pte_offset_map(pmd, address);
|
| 2765 | if (!pte)
|
| 2766 | goto again;
|
| 2767 |
|
| 2768 | |
| 2769 | * Lockless access: we're in a wait_event so it's ok if it
|
| 2770 | * changes under us.
|
| 2771 |
|
| 2772 | ptent = ptep_get(pte);
|
| 2773 |
|
| 2774 | ret = true;
|
| 2775 |
|
| 2776 | if (pte_none(ptent))
|
| 2777 | goto out;
|
| 2778 |
|
| 2779 | if (pte_is_uffd_marker(ptent))
|
| 2780 | goto out;
|
| 2781 | |
| 2782 | * Concurrent swap-out / migration may have replaced the present PTE
|
| 2783 | * with a non-marker swap entry between fault delivery and this
|
| 2784 | * lockless re-check. pte_write() on a swap entry decodes random
|
| 2785 | * offset bits, so gate it on pte_present(). The page-in path will
|
| 2786 | * re-deliver the fault if it still needs userspace.
|
| 2787 |
|
| 2788 | if (!pte_present(ptent)) {
|
| 2789 | ret = false;
|
| 2790 | goto out;
|
| 2791 | }
|
| 2792 | |
| 2793 | * If VMA has UFFD WP faults enabled and WP fault, wait for userspace to
|
| 2794 | * resolve the fault.
|
| 2795 |
|
| 2796 | if (!pte_write(ptent) && (reason & VM_UFFD_WP))
|
| 2797 | goto out;
|
| 2798 | |
| 2799 | * PTE is still RW-protected (protnone with uffd bit), wait for
|
| 2800 | * userspace to resolve. Plain PROT_NONE without the marker is not
|
| 2801 | * an RWP fault.
|
| 2802 |
|
| 2803 | if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP))
|
| 2804 | goto out;
|
| 2805 |
|
| 2806 | ret = false;
|
| 2807 | out:
|
| 2808 | pte_unmap(pte);
|
| 2809 | return ret;
|
| 2810 | }
|
| 2811 |
|
| 2812 | static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)
|
| 2813 | {
|
| 2814 | if (flags & FAULT_FLAG_INTERRUPTIBLE)
|
| 2815 | return TASK_INTERRUPTIBLE;
|
| 2816 |
|
| 2817 | if (flags & FAULT_FLAG_KILLABLE)
|
| 2818 | return TASK_KILLABLE;
|
| 2819 |
|
| 2820 | return TASK_UNINTERRUPTIBLE;
|
| 2821 | }
|
| 2822 |
|
| 2823 | |
| 2824 | * The locking rules involved in returning VM_FAULT_RETRY depending on
|
| 2825 | * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
|
| 2826 | * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
|
| 2827 | * recommendation in __lock_page_or_retry is not an understatement.
|
| 2828 | *
|
| 2829 | * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
|
| 2830 | * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
|
| 2831 | * not set.
|
| 2832 | *
|
| 2833 | * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
|
| 2834 | * set, VM_FAULT_RETRY can still be returned if and only if there are
|
| 2835 | * fatal_signal_pending()s, and the mmap_lock must be released before
|
| 2836 | * returning it.
|
| 2837 |
|
| 2838 | vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
|
| 2839 | {
|
| 2840 | struct vm_area_struct *vma = vmf->vma;
|
| 2841 | struct mm_struct *mm = vma->vm_mm;
|
| 2842 | struct userfaultfd_ctx *ctx;
|
| 2843 | struct userfaultfd_wait_queue uwq;
|
| 2844 | vm_fault_t ret = VM_FAULT_SIGBUS;
|
| 2845 | bool must_wait;
|
| 2846 | unsigned int blocking_state;
|
| 2847 |
|
| 2848 | |
| 2849 | * We don't do userfault handling for the final child pid update
|
| 2850 | * and when coredumping (faults triggered by get_dump_page()).
|
| 2851 |
|
| 2852 | if (current->flags & (PF_EXITING|PF_DUMPCORE))
|
| 2853 | goto out;
|
| 2854 |
|
| 2855 | assert_fault_locked(vmf);
|
| 2856 |
|
| 2857 | ctx = vma->vm_userfaultfd_ctx.ctx;
|
| 2858 | if (!ctx)
|
| 2859 | goto out;
|
| 2860 |
|
| 2861 | VM_WARN_ON_ONCE(ctx->mm != mm);
|
| 2862 |
|
| 2863 |
|
| 2864 | VM_WARN_ON_ONCE(reason & ~__VM_UFFD_FLAGS);
|
| 2865 |
|
| 2866 | VM_WARN_ON_ONCE(!reason || (reason & (reason - 1)));
|
| 2867 |
|
| 2868 | if (ctx->features & UFFD_FEATURE_SIGBUS)
|
| 2869 | goto out;
|
| 2870 | if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))
|
| 2871 | goto out;
|
| 2872 |
|
| 2873 | |
| 2874 | * Check that we can return VM_FAULT_RETRY.
|
| 2875 | *
|
| 2876 | * NOTE: it should become possible to return VM_FAULT_RETRY
|
| 2877 | * even if FAULT_FLAG_TRIED is set without leading to gup()
|
| 2878 | * -EBUSY failures, if the userfaultfd is to be extended for
|
| 2879 | * VM_UFFD_WP tracking and we intend to arm the userfault
|
| 2880 | * without first stopping userland access to the memory. For
|
| 2881 | * VM_UFFD_MISSING userfaults this is enough for now.
|
| 2882 |
|
| 2883 | if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
|
| 2884 | |
| 2885 | * Validate the invariant that nowait must allow retry
|
| 2886 | * to be sure not to return SIGBUS erroneously on
|
| 2887 | * nowait invocations.
|
| 2888 |
|
| 2889 | VM_WARN_ON_ONCE(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
|
| 2890 | #ifdef CONFIG_DEBUG_VM
|
| 2891 | if (printk_ratelimit()) {
|
| 2892 | pr_warn("FAULT_FLAG_ALLOW_RETRY missing %x\n",
|
| 2893 | vmf->flags);
|
| 2894 | dump_stack();
|
| 2895 | }
|
| 2896 | #endif
|
| 2897 | goto out;
|
| 2898 | }
|
| 2899 |
|
| 2900 | |
| 2901 | * Handle nowait, not much to do other than tell it to retry
|
| 2902 | * and wait.
|
| 2903 |
|
| 2904 | ret = VM_FAULT_RETRY;
|
| 2905 | if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
|
| 2906 | goto out;
|
| 2907 |
|
| 2908 | if (unlikely(READ_ONCE(ctx->released))) {
|
| 2909 | |
| 2910 | * If a concurrent release is detected, do not return
|
| 2911 | * VM_FAULT_SIGBUS or VM_FAULT_NOPAGE, but instead always
|
| 2912 | * return VM_FAULT_RETRY with lock released proactively.
|
| 2913 | *
|
| 2914 | * If we were to return VM_FAULT_SIGBUS here, the non
|
| 2915 | * cooperative manager would be instead forced to
|
| 2916 | * always call UFFDIO_UNREGISTER before it can safely
|
| 2917 | * close the uffd, to avoid involuntary SIGBUS triggered.
|
| 2918 | *
|
| 2919 | * If we were to return VM_FAULT_NOPAGE, it would work for
|
| 2920 | * the fault path, in which the lock will be released
|
| 2921 | * later. However for GUP, faultin_page() does nothing
|
| 2922 | * special on NOPAGE, so GUP would spin retrying without
|
| 2923 | * releasing the mmap read lock, causing possible livelock.
|
| 2924 | *
|
| 2925 | * Here only VM_FAULT_RETRY would make sure the mmap lock
|
| 2926 | * be released immediately, so that the thread concurrently
|
| 2927 | * releasing the userfault would always make progress.
|
| 2928 |
|
| 2929 | release_fault_lock(vmf);
|
| 2930 | goto out;
|
| 2931 | }
|
| 2932 |
|
| 2933 |
|
| 2934 | userfaultfd_ctx_get(ctx);
|
| 2935 |
|
| 2936 | init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
|
| 2937 | uwq.wq.private = current;
|
| 2938 | uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags,
|
| 2939 | reason, ctx->features);
|
| 2940 | uwq.ctx = ctx;
|
| 2941 | uwq.waken = false;
|
| 2942 |
|
| 2943 | blocking_state = userfaultfd_get_blocking_state(vmf->flags);
|
| 2944 |
|
| 2945 | |
| 2946 | * Take the vma lock now, in order to safely call
|
| 2947 | * userfaultfd_huge_must_wait() later. Since acquiring the
|
| 2948 | * (sleepable) vma lock can modify the current task state, that
|
| 2949 | * must be before explicitly calling set_current_state().
|
| 2950 |
|
| 2951 | if (is_vm_hugetlb_page(vma))
|
| 2952 | hugetlb_vma_lock_read(vma);
|
| 2953 |
|
| 2954 | spin_lock_irq(&ctx->fault_pending_wqh.lock);
|
| 2955 | |
| 2956 | * After the __add_wait_queue the uwq is visible to userland
|
| 2957 | * through poll/read().
|
| 2958 |
|
| 2959 | __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
|
| 2960 | |
| 2961 | * The smp_mb() after __set_current_state prevents the reads
|
| 2962 | * following the spin_unlock to happen before the list_add in
|
| 2963 | * __add_wait_queue.
|
| 2964 |
|
| 2965 | set_current_state(blocking_state);
|
| 2966 | spin_unlock_irq(&ctx->fault_pending_wqh.lock);
|
| 2967 |
|
| 2968 | if (is_vm_hugetlb_page(vma)) {
|
| 2969 | must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason);
|
| 2970 | hugetlb_vma_unlock_read(vma);
|
| 2971 | } else {
|
| 2972 | must_wait = userfaultfd_must_wait(ctx, vmf, reason);
|
| 2973 | }
|
| 2974 |
|
| 2975 | release_fault_lock(vmf);
|
| 2976 |
|
| 2977 | if (likely(must_wait && !READ_ONCE(ctx->released))) {
|
| 2978 | wake_up_poll(&ctx->fd_wqh, EPOLLIN);
|
| 2979 | schedule();
|
| 2980 | }
|
| 2981 |
|
| 2982 | __set_current_state(TASK_RUNNING);
|
| 2983 |
|
| 2984 | |
| 2985 | * Here we race with the list_del; list_add in
|
| 2986 | * userfaultfd_ctx_read(), however because we don't ever run
|
| 2987 | * list_del_init() to refile across the two lists, the prev
|
| 2988 | * and next pointers will never point to self. list_add also
|
| 2989 | * would never let any of the two pointers to point to
|
| 2990 | * self. So list_empty_careful won't risk to see both pointers
|
| 2991 | * pointing to self at any time during the list refile. The
|
| 2992 | * only case where list_del_init() is called is the full
|
| 2993 | * removal in the wake function and there we don't re-list_add
|
| 2994 | * and it's fine not to block on the spinlock. The uwq on this
|
| 2995 | * kernel stack can be released after the list_del_init.
|
| 2996 |
|
| 2997 | if (!list_empty_careful(&uwq.wq.entry)) {
|
| 2998 | spin_lock_irq(&ctx->fault_pending_wqh.lock);
|
| 2999 | |
| 3000 | * No need of list_del_init(), the uwq on the stack
|
| 3001 | * will be freed shortly anyway.
|
| 3002 |
|
| 3003 | list_del(&uwq.wq.entry);
|
| 3004 | spin_unlock_irq(&ctx->fault_pending_wqh.lock);
|
| 3005 | }
|
| 3006 |
|
| 3007 | |
| 3008 | * ctx may go away after this if the userfault pseudo fd is
|
| 3009 | * already released.
|
| 3010 |
|
| 3011 | userfaultfd_ctx_put(ctx);
|
| 3012 |
|
| 3013 | out:
|
| 3014 | return ret;
|
| 3015 | }
|
| 3016 |
|
| 3017 | static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
|
| 3018 | struct userfaultfd_wait_queue *ewq)
|
| 3019 | {
|
| 3020 | struct userfaultfd_ctx *release_new_ctx;
|
| 3021 |
|
| 3022 | if (WARN_ON_ONCE(current->flags & PF_EXITING))
|
| 3023 | goto out;
|
| 3024 |
|
| 3025 | ewq->ctx = ctx;
|
| 3026 | init_waitqueue_entry(&ewq->wq, current);
|
| 3027 | release_new_ctx = NULL;
|
| 3028 |
|
| 3029 | spin_lock_irq(&ctx->event_wqh.lock);
|
| 3030 | |
| 3031 | * After the __add_wait_queue the uwq is visible to userland
|
| 3032 | * through poll/read().
|
| 3033 |
|
| 3034 | __add_wait_queue(&ctx->event_wqh, &ewq->wq);
|
| 3035 | for (;;) {
|
| 3036 | set_current_state(TASK_KILLABLE);
|
| 3037 | if (ewq->msg.event == 0)
|
| 3038 | break;
|
| 3039 | if (READ_ONCE(ctx->released) ||
|
| 3040 | fatal_signal_pending(current)) {
|
| 3041 | |
| 3042 | * &ewq->wq may be queued in fork_event, but
|
| 3043 | * __remove_wait_queue ignores the head
|
| 3044 | * parameter. It would be a problem if it
|
| 3045 | * didn't.
|
| 3046 |
|
| 3047 | __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
|
| 3048 | if (ewq->msg.event == UFFD_EVENT_FORK) {
|
| 3049 | struct userfaultfd_ctx *new;
|
| 3050 |
|
| 3051 | new = (struct userfaultfd_ctx *)
|
| 3052 | (unsigned long)
|
| 3053 | ewq->msg.arg.reserved.reserved1;
|
| 3054 | release_new_ctx = new;
|
| 3055 | }
|
| 3056 | break;
|
| 3057 | }
|
| 3058 |
|
| 3059 | spin_unlock_irq(&ctx->event_wqh.lock);
|
| 3060 |
|
| 3061 | wake_up_poll(&ctx->fd_wqh, EPOLLIN);
|
| 3062 | schedule();
|
| 3063 |
|
| 3064 | spin_lock_irq(&ctx->event_wqh.lock);
|
| 3065 | }
|
| 3066 | __set_current_state(TASK_RUNNING);
|
| 3067 | spin_unlock_irq(&ctx->event_wqh.lock);
|
| 3068 |
|
| 3069 | if (release_new_ctx) {
|
| 3070 | userfaultfd_release_new(release_new_ctx);
|
| 3071 | userfaultfd_ctx_put(release_new_ctx);
|
| 3072 | }
|
| 3073 |
|
| 3074 | |
| 3075 | * ctx may go away after this if the userfault pseudo fd is
|
| 3076 | * already released.
|
| 3077 |
|
| 3078 | out:
|
| 3079 | atomic_dec(&ctx->mmap_changing);
|
| 3080 | VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0);
|
| 3081 | userfaultfd_ctx_put(ctx);
|
| 3082 | }
|
| 3083 |
|
| 3084 | static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
|
| 3085 | struct userfaultfd_wait_queue *ewq)
|
| 3086 | {
|
| 3087 | ewq->msg.event = 0;
|
| 3088 | wake_up_locked(&ctx->event_wqh);
|
| 3089 | __remove_wait_queue(&ctx->event_wqh, &ewq->wq);
|
| 3090 | }
|
| 3091 |
|
| 3092 | int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
|
| 3093 | {
|
| 3094 | struct userfaultfd_ctx *ctx = NULL, *octx;
|
| 3095 | struct userfaultfd_fork_ctx *fctx;
|
| 3096 |
|
| 3097 | octx = vma->vm_userfaultfd_ctx.ctx;
|
| 3098 | if (!octx)
|
| 3099 | return 0;
|
| 3100 |
|
| 3101 | if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) {
|
| 3102 | userfaultfd_reset_ctx(vma);
|
| 3103 | return 0;
|
| 3104 | }
|
| 3105 |
|
| 3106 | list_for_each_entry(fctx, fcs, list)
|
| 3107 | if (fctx->orig == octx) {
|
| 3108 | ctx = fctx->new;
|
| 3109 | break;
|
| 3110 | }
|
| 3111 |
|
| 3112 | if (!ctx) {
|
| 3113 | fctx = kmalloc_obj(*fctx);
|
| 3114 | if (!fctx)
|
| 3115 | return -ENOMEM;
|
| 3116 |
|
| 3117 | ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
|
| 3118 | if (!ctx) {
|
| 3119 | kfree(fctx);
|
| 3120 | return -ENOMEM;
|
| 3121 | }
|
| 3122 |
|
| 3123 | refcount_set(&ctx->refcount, 1);
|
| 3124 | ctx->flags = octx->flags;
|
| 3125 | ctx->features = octx->features;
|
| 3126 | ctx->released = false;
|
| 3127 | init_rwsem(&ctx->map_changing_lock);
|
| 3128 | atomic_set(&ctx->mmap_changing, 0);
|
| 3129 | ctx->mm = vma->vm_mm;
|
| 3130 | mmgrab(ctx->mm);
|
| 3131 |
|
| 3132 | userfaultfd_ctx_get(octx);
|
| 3133 | down_write(&octx->map_changing_lock);
|
| 3134 | atomic_inc(&octx->mmap_changing);
|
| 3135 | up_write(&octx->map_changing_lock);
|
| 3136 | fctx->orig = octx;
|
| 3137 | fctx->new = ctx;
|
| 3138 | list_add_tail(&fctx->list, fcs);
|
| 3139 | }
|
| 3140 |
|
| 3141 | vma->vm_userfaultfd_ctx.ctx = ctx;
|
| 3142 | return 0;
|
| 3143 | }
|
| 3144 |
|
| 3145 | static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
|
| 3146 | {
|
| 3147 | struct userfaultfd_ctx *ctx = fctx->orig;
|
| 3148 | struct userfaultfd_wait_queue ewq;
|
| 3149 |
|
| 3150 | msg_init(&ewq.msg);
|
| 3151 |
|
| 3152 | ewq.msg.event = UFFD_EVENT_FORK;
|
| 3153 | ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
|
| 3154 |
|
| 3155 | userfaultfd_event_wait_completion(ctx, &ewq);
|
| 3156 | }
|
| 3157 |
|
| 3158 | void dup_userfaultfd_complete(struct list_head *fcs)
|
| 3159 | {
|
| 3160 | struct userfaultfd_fork_ctx *fctx, *n;
|
| 3161 |
|
| 3162 | list_for_each_entry_safe(fctx, n, fcs, list) {
|
| 3163 | dup_fctx(fctx);
|
| 3164 | list_del(&fctx->list);
|
| 3165 | kfree(fctx);
|
| 3166 | }
|
| 3167 | }
|
| 3168 |
|
| 3169 | void dup_userfaultfd_fail(struct list_head *fcs)
|
| 3170 | {
|
| 3171 | struct userfaultfd_fork_ctx *fctx, *n;
|
| 3172 |
|
| 3173 | |
| 3174 | * An error has occurred on fork, we will tear memory down, but have
|
| 3175 | * allocated memory for fctx's and raised reference counts for both the
|
| 3176 | * original and child contexts (and on the mm for each as a result).
|
| 3177 | *
|
| 3178 | * These would ordinarily be taken care of by a user handling the event,
|
| 3179 | * but we are no longer doing so, so manually clean up here.
|
| 3180 | *
|
| 3181 | * mm tear down will take care of cleaning up VMA contexts.
|
| 3182 |
|
| 3183 | list_for_each_entry_safe(fctx, n, fcs, list) {
|
| 3184 | struct userfaultfd_ctx *octx = fctx->orig;
|
| 3185 | struct userfaultfd_ctx *ctx = fctx->new;
|
| 3186 |
|
| 3187 | atomic_dec(&octx->mmap_changing);
|
| 3188 | VM_WARN_ON_ONCE(atomic_read(&octx->mmap_changing) < 0);
|
| 3189 | userfaultfd_ctx_put(octx);
|
| 3190 | userfaultfd_ctx_put(ctx);
|
| 3191 |
|
| 3192 | list_del(&fctx->list);
|
| 3193 | kfree(fctx);
|
| 3194 | }
|
| 3195 | }
|
| 3196 |
|
| 3197 | void mremap_userfaultfd_prep(struct vm_area_struct *vma,
|
| 3198 | struct vm_userfaultfd_ctx *vm_ctx)
|
| 3199 | {
|
| 3200 | struct userfaultfd_ctx *ctx;
|
| 3201 |
|
| 3202 | ctx = vma->vm_userfaultfd_ctx.ctx;
|
| 3203 |
|
| 3204 | if (!ctx)
|
| 3205 | return;
|
| 3206 |
|
| 3207 | if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
|
| 3208 | vm_ctx->ctx = ctx;
|
| 3209 | userfaultfd_ctx_get(ctx);
|
| 3210 | down_write(&ctx->map_changing_lock);
|
| 3211 | atomic_inc(&ctx->mmap_changing);
|
| 3212 | up_write(&ctx->map_changing_lock);
|
| 3213 | } else {
|
| 3214 |
|
| 3215 | userfaultfd_reset_ctx(vma);
|
| 3216 | }
|
| 3217 | }
|
| 3218 |
|
| 3219 | void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
|
| 3220 | unsigned long from, unsigned long to,
|
| 3221 | unsigned long len)
|
| 3222 | {
|
| 3223 | struct userfaultfd_ctx *ctx = vm_ctx->ctx;
|
| 3224 | struct userfaultfd_wait_queue ewq;
|
| 3225 |
|
| 3226 | if (!ctx)
|
| 3227 | return;
|
| 3228 |
|
| 3229 | msg_init(&ewq.msg);
|
| 3230 |
|
| 3231 | ewq.msg.event = UFFD_EVENT_REMAP;
|
| 3232 | ewq.msg.arg.remap.from = from;
|
| 3233 | ewq.msg.arg.remap.to = to;
|
| 3234 | ewq.msg.arg.remap.len = len;
|
| 3235 |
|
| 3236 | userfaultfd_event_wait_completion(ctx, &ewq);
|
| 3237 | }
|
| 3238 |
|
| 3239 | void mremap_userfaultfd_fail(struct vm_userfaultfd_ctx *vm_ctx)
|
| 3240 | {
|
| 3241 | struct userfaultfd_ctx *ctx = vm_ctx->ctx;
|
| 3242 |
|
| 3243 | if (!ctx)
|
| 3244 | return;
|
| 3245 |
|
| 3246 | atomic_dec(&ctx->mmap_changing);
|
| 3247 | VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0);
|
| 3248 | userfaultfd_ctx_put(ctx);
|
| 3249 | }
|
| 3250 |
|
| 3251 | bool userfaultfd_remove(struct vm_area_struct *vma,
|
| 3252 | unsigned long start, unsigned long end)
|
| 3253 | {
|
| 3254 | struct mm_struct *mm = vma->vm_mm;
|
| 3255 | struct userfaultfd_ctx *ctx;
|
| 3256 | struct userfaultfd_wait_queue ewq;
|
| 3257 |
|
| 3258 | ctx = vma->vm_userfaultfd_ctx.ctx;
|
| 3259 | if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
|
| 3260 | return true;
|
| 3261 |
|
| 3262 | userfaultfd_ctx_get(ctx);
|
| 3263 | down_write(&ctx->map_changing_lock);
|
| 3264 | atomic_inc(&ctx->mmap_changing);
|
| 3265 | up_write(&ctx->map_changing_lock);
|
| 3266 | mmap_read_unlock(mm);
|
| 3267 |
|
| 3268 | msg_init(&ewq.msg);
|
| 3269 |
|
| 3270 | ewq.msg.event = UFFD_EVENT_REMOVE;
|
| 3271 | ewq.msg.arg.remove.start = start;
|
| 3272 | ewq.msg.arg.remove.end = end;
|
| 3273 |
|
| 3274 | userfaultfd_event_wait_completion(ctx, &ewq);
|
| 3275 |
|
| 3276 | return false;
|
| 3277 | }
|
| 3278 |
|
| 3279 | static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
|
| 3280 | unsigned long start, unsigned long end)
|
| 3281 | {
|
| 3282 | struct userfaultfd_unmap_ctx *unmap_ctx;
|
| 3283 |
|
| 3284 | list_for_each_entry(unmap_ctx, unmaps, list)
|
| 3285 | if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
|
| 3286 | unmap_ctx->end == end)
|
| 3287 | return true;
|
| 3288 |
|
| 3289 | return false;
|
| 3290 | }
|
| 3291 |
|
| 3292 | int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start,
|
| 3293 | unsigned long end, struct list_head *unmaps)
|
| 3294 | {
|
| 3295 | struct userfaultfd_unmap_ctx *unmap_ctx;
|
| 3296 | struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
|
| 3297 |
|
| 3298 | if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
|
| 3299 | has_unmap_ctx(ctx, unmaps, start, end))
|
| 3300 | return 0;
|
| 3301 |
|
| 3302 | unmap_ctx = kzalloc_obj(*unmap_ctx);
|
| 3303 | if (!unmap_ctx)
|
| 3304 | return -ENOMEM;
|
| 3305 |
|
| 3306 | userfaultfd_ctx_get(ctx);
|
| 3307 | down_write(&ctx->map_changing_lock);
|
| 3308 | atomic_inc(&ctx->mmap_changing);
|
| 3309 | up_write(&ctx->map_changing_lock);
|
| 3310 | unmap_ctx->ctx = ctx;
|
| 3311 | unmap_ctx->start = start;
|
| 3312 | unmap_ctx->end = end;
|
| 3313 | list_add_tail(&unmap_ctx->list, unmaps);
|
| 3314 |
|
| 3315 | return 0;
|
| 3316 | }
|
| 3317 |
|
| 3318 | void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
|
| 3319 | {
|
| 3320 | struct userfaultfd_unmap_ctx *ctx, *n;
|
| 3321 | struct userfaultfd_wait_queue ewq;
|
| 3322 |
|
| 3323 | list_for_each_entry_safe(ctx, n, uf, list) {
|
| 3324 | msg_init(&ewq.msg);
|
| 3325 |
|
| 3326 | ewq.msg.event = UFFD_EVENT_UNMAP;
|
| 3327 | ewq.msg.arg.remove.start = ctx->start;
|
| 3328 | ewq.msg.arg.remove.end = ctx->end;
|
| 3329 |
|
| 3330 | userfaultfd_event_wait_completion(ctx->ctx, &ewq);
|
| 3331 |
|
| 3332 | list_del(&ctx->list);
|
| 3333 | kfree(ctx);
|
| 3334 | }
|
| 3335 | }
|
| 3336 |
|
| 3337 | static int userfaultfd_release(struct inode *inode, struct file *file)
|
| 3338 | {
|
| 3339 | struct userfaultfd_ctx *ctx = file->private_data;
|
| 3340 | struct mm_struct *mm = ctx->mm;
|
| 3341 |
|
| 3342 | struct userfaultfd_wake_range range = { .len = 0, };
|
| 3343 |
|
| 3344 | WRITE_ONCE(ctx->released, true);
|
| 3345 |
|
| 3346 | userfaultfd_release_all(mm, ctx);
|
| 3347 |
|
| 3348 | |
| 3349 | * After no new page faults can wait on this fault_*wqh, flush
|
| 3350 | * the last page faults that may have been already waiting on
|
| 3351 | * the fault_*wqh.
|
| 3352 |
|
| 3353 | spin_lock_irq(&ctx->fault_pending_wqh.lock);
|
| 3354 | __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
|
| 3355 | __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
|
| 3356 | spin_unlock_irq(&ctx->fault_pending_wqh.lock);
|
| 3357 |
|
| 3358 |
|
| 3359 | wake_up_all(&ctx->event_wqh);
|
| 3360 |
|
| 3361 | wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
|
| 3362 | userfaultfd_ctx_put(ctx);
|
| 3363 | return 0;
|
| 3364 | }
|
| 3365 |
|
| 3366 |
|
| 3367 | static inline struct userfaultfd_wait_queue *find_userfault_in(
|
| 3368 | wait_queue_head_t *wqh)
|
| 3369 | {
|
| 3370 | wait_queue_entry_t *wq;
|
| 3371 | struct userfaultfd_wait_queue *uwq;
|
| 3372 |
|
| 3373 | lockdep_assert_held(&wqh->lock);
|
| 3374 |
|
| 3375 | uwq = NULL;
|
| 3376 | if (!waitqueue_active(wqh))
|
| 3377 | goto out;
|
| 3378 |
|
| 3379 | wq = list_last_entry(&wqh->head, typeof(*wq), entry);
|
| 3380 | uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
|
| 3381 | out:
|
| 3382 | return uwq;
|
| 3383 | }
|
| 3384 |
|
| 3385 | static inline struct userfaultfd_wait_queue *find_userfault(
|
| 3386 | struct userfaultfd_ctx *ctx)
|
| 3387 | {
|
| 3388 | return find_userfault_in(&ctx->fault_pending_wqh);
|
| 3389 | }
|
| 3390 |
|
| 3391 | static inline struct userfaultfd_wait_queue *find_userfault_evt(
|
| 3392 | struct userfaultfd_ctx *ctx)
|
| 3393 | {
|
| 3394 | return find_userfault_in(&ctx->event_wqh);
|
| 3395 | }
|
| 3396 |
|
| 3397 | static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
|
| 3398 | {
|
| 3399 | struct userfaultfd_ctx *ctx = file->private_data;
|
| 3400 | __poll_t ret;
|
| 3401 |
|
| 3402 | poll_wait(file, &ctx->fd_wqh, wait);
|
| 3403 |
|
| 3404 | if (!userfaultfd_is_initialized(ctx))
|
| 3405 | return EPOLLERR;
|
| 3406 |
|
| 3407 | |
| 3408 | * poll() never guarantees that read won't block.
|
| 3409 | * userfaults can be waken before they're read().
|
| 3410 |
|
| 3411 | if (unlikely(!(file->f_flags & O_NONBLOCK)))
|
| 3412 | return EPOLLERR;
|
| 3413 | |
| 3414 | * lockless access to see if there are pending faults
|
| 3415 | * __pollwait last action is the add_wait_queue but
|
| 3416 | * the spin_unlock would allow the waitqueue_active to
|
| 3417 | * pass above the actual list_add inside
|
| 3418 | * add_wait_queue critical section. So use a full
|
| 3419 | * memory barrier to serialize the list_add write of
|
| 3420 | * add_wait_queue() with the waitqueue_active read
|
| 3421 | * below.
|
| 3422 |
|
| 3423 | ret = 0;
|
| 3424 | smp_mb();
|
| 3425 | if (waitqueue_active(&ctx->fault_pending_wqh))
|
| 3426 | ret = EPOLLIN;
|
| 3427 | else if (waitqueue_active(&ctx->event_wqh))
|
| 3428 | ret = EPOLLIN;
|
| 3429 |
|
| 3430 | return ret;
|
| 3431 | }
|
| 3432 |
|
| 3433 | static const struct file_operations userfaultfd_fops;
|
| 3434 |
|
| 3435 | static int resolve_userfault_fork(struct userfaultfd_ctx *new,
|
| 3436 | struct inode *inode,
|
| 3437 | struct uffd_msg *msg)
|
| 3438 | {
|
| 3439 | int fd;
|
| 3440 |
|
| 3441 | fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new,
|
| 3442 | O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
|
| 3443 | if (fd < 0)
|
| 3444 | return fd;
|
| 3445 |
|
| 3446 | msg->arg.reserved.reserved1 = 0;
|
| 3447 | msg->arg.fork.ufd = fd;
|
| 3448 | return 0;
|
| 3449 | }
|
| 3450 |
|
| 3451 | static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
|
| 3452 | struct uffd_msg *msg, struct inode *inode)
|
| 3453 | {
|
| 3454 | ssize_t ret;
|
| 3455 | DECLARE_WAITQUEUE(wait, current);
|
| 3456 | struct userfaultfd_wait_queue *uwq;
|
| 3457 | |
| 3458 | * Handling fork event requires sleeping operations, so
|
| 3459 | * we drop the event_wqh lock, then do these ops, then
|
| 3460 | * lock it back and wake up the waiter. While the lock is
|
| 3461 | * dropped the ewq may go away so we keep track of it
|
| 3462 | * carefully.
|
| 3463 |
|
| 3464 | LIST_HEAD(fork_event);
|
| 3465 | struct userfaultfd_ctx *fork_nctx = NULL;
|
| 3466 |
|
| 3467 |
|
| 3468 | spin_lock_irq(&ctx->fd_wqh.lock);
|
| 3469 | __add_wait_queue(&ctx->fd_wqh, &wait);
|
| 3470 | for (;;) {
|
| 3471 | set_current_state(TASK_INTERRUPTIBLE);
|
| 3472 | spin_lock(&ctx->fault_pending_wqh.lock);
|
| 3473 | uwq = find_userfault(ctx);
|
| 3474 | if (uwq) {
|
| 3475 | |
| 3476 | * Use a seqcount to repeat the lockless check
|
| 3477 | * in wake_userfault() to avoid missing
|
| 3478 | * wakeups because during the refile both
|
| 3479 | * waitqueue could become empty if this is the
|
| 3480 | * only userfault.
|
| 3481 |
|
| 3482 | write_seqcount_begin(&ctx->refile_seq);
|
| 3483 |
|
| 3484 | |
| 3485 | * The fault_pending_wqh.lock prevents the uwq
|
| 3486 | * to disappear from under us.
|
| 3487 | *
|
| 3488 | * Refile this userfault from
|
| 3489 | * fault_pending_wqh to fault_wqh, it's not
|
| 3490 | * pending anymore after we read it.
|
| 3491 | *
|
| 3492 | * Use list_del() by hand (as
|
| 3493 | * userfaultfd_wake_function also uses
|
| 3494 | * list_del_init() by hand) to be sure nobody
|
| 3495 | * changes __remove_wait_queue() to use
|
| 3496 | * list_del_init() in turn breaking the
|
| 3497 | * !list_empty_careful() check in
|
| 3498 | * handle_userfault(). The uwq->wq.head list
|
| 3499 | * must never be empty at any time during the
|
| 3500 | * refile, or the waitqueue could disappear
|
| 3501 | * from under us. The "wait_queue_head_t"
|
| 3502 | * parameter of __remove_wait_queue() is unused
|
| 3503 | * anyway.
|
| 3504 |
|
| 3505 | list_del(&uwq->wq.entry);
|
| 3506 | add_wait_queue(&ctx->fault_wqh, &uwq->wq);
|
| 3507 |
|
| 3508 | write_seqcount_end(&ctx->refile_seq);
|
| 3509 |
|
| 3510 |
|
| 3511 | *msg = uwq->msg;
|
| 3512 | spin_unlock(&ctx->fault_pending_wqh.lock);
|
| 3513 | ret = 0;
|
| 3514 | break;
|
| 3515 | }
|
| 3516 | spin_unlock(&ctx->fault_pending_wqh.lock);
|
| 3517 |
|
| 3518 | spin_lock(&ctx->event_wqh.lock);
|
| 3519 | uwq = find_userfault_evt(ctx);
|
| 3520 | if (uwq) {
|
| 3521 | *msg = uwq->msg;
|
| 3522 |
|
| 3523 | if (uwq->msg.event == UFFD_EVENT_FORK) {
|
| 3524 | fork_nctx = (struct userfaultfd_ctx *)
|
| 3525 | (unsigned long)
|
| 3526 | uwq->msg.arg.reserved.reserved1;
|
| 3527 | list_move(&uwq->wq.entry, &fork_event);
|
| 3528 | |
| 3529 | * fork_nctx can be freed as soon as
|
| 3530 | * we drop the lock, unless we take a
|
| 3531 | * reference on it.
|
| 3532 |
|
| 3533 | userfaultfd_ctx_get(fork_nctx);
|
| 3534 | spin_unlock(&ctx->event_wqh.lock);
|
| 3535 | ret = 0;
|
| 3536 | break;
|
| 3537 | }
|
| 3538 |
|
| 3539 | userfaultfd_event_complete(ctx, uwq);
|
| 3540 | spin_unlock(&ctx->event_wqh.lock);
|
| 3541 | ret = 0;
|
| 3542 | break;
|
| 3543 | }
|
| 3544 | spin_unlock(&ctx->event_wqh.lock);
|
| 3545 |
|
| 3546 | if (signal_pending(current)) {
|
| 3547 | ret = -ERESTARTSYS;
|
| 3548 | break;
|
| 3549 | }
|
| 3550 | if (no_wait) {
|
| 3551 | ret = -EAGAIN;
|
| 3552 | break;
|
| 3553 | }
|
| 3554 | spin_unlock_irq(&ctx->fd_wqh.lock);
|
| 3555 | schedule();
|
| 3556 | spin_lock_irq(&ctx->fd_wqh.lock);
|
| 3557 | }
|
| 3558 | __remove_wait_queue(&ctx->fd_wqh, &wait);
|
| 3559 | __set_current_state(TASK_RUNNING);
|
| 3560 | spin_unlock_irq(&ctx->fd_wqh.lock);
|
| 3561 |
|
| 3562 | if (!ret && msg->event == UFFD_EVENT_FORK) {
|
| 3563 | ret = resolve_userfault_fork(fork_nctx, inode, msg);
|
| 3564 | spin_lock_irq(&ctx->event_wqh.lock);
|
| 3565 | if (!list_empty(&fork_event)) {
|
| 3566 | |
| 3567 | * The fork thread didn't abort, so we can
|
| 3568 | * drop the temporary refcount.
|
| 3569 |
|
| 3570 | userfaultfd_ctx_put(fork_nctx);
|
| 3571 |
|
| 3572 | uwq = list_first_entry(&fork_event,
|
| 3573 | typeof(*uwq),
|
| 3574 | wq.entry);
|
| 3575 | |
| 3576 | * If fork_event list wasn't empty and in turn
|
| 3577 | * the event wasn't already released by fork
|
| 3578 | * (the event is allocated on fork kernel
|
| 3579 | * stack), put the event back to its place in
|
| 3580 | * the event_wq. fork_event head will be freed
|
| 3581 | * as soon as we return so the event cannot
|
| 3582 | * stay queued there no matter the current
|
| 3583 | * "ret" value.
|
| 3584 |
|
| 3585 | list_del(&uwq->wq.entry);
|
| 3586 | __add_wait_queue(&ctx->event_wqh, &uwq->wq);
|
| 3587 |
|
| 3588 | |
| 3589 | * Leave the event in the waitqueue and report
|
| 3590 | * error to userland if we failed to resolve
|
| 3591 | * the userfault fork.
|
| 3592 |
|
| 3593 | if (likely(!ret))
|
| 3594 | userfaultfd_event_complete(ctx, uwq);
|
| 3595 | } else {
|
| 3596 | |
| 3597 | * Here the fork thread aborted and the
|
| 3598 | * refcount from the fork thread on fork_nctx
|
| 3599 | * has already been released. We still hold
|
| 3600 | * the reference we took before releasing the
|
| 3601 | * lock above. If resolve_userfault_fork
|
| 3602 | * failed we've to drop it because the
|
| 3603 | * fork_nctx has to be freed in such case. If
|
| 3604 | * it succeeded we'll hold it because the new
|
| 3605 | * uffd references it.
|
| 3606 |
|
| 3607 | if (ret)
|
| 3608 | userfaultfd_ctx_put(fork_nctx);
|
| 3609 | }
|
| 3610 | spin_unlock_irq(&ctx->event_wqh.lock);
|
| 3611 | }
|
| 3612 |
|
| 3613 | return ret;
|
| 3614 | }
|
| 3615 |
|
| 3616 | static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to)
|
| 3617 | {
|
| 3618 | struct file *file = iocb->ki_filp;
|
| 3619 | struct userfaultfd_ctx *ctx = file->private_data;
|
| 3620 | ssize_t _ret, ret = 0;
|
| 3621 | struct uffd_msg msg;
|
| 3622 | struct inode *inode = file_inode(file);
|
| 3623 | bool no_wait;
|
| 3624 |
|
| 3625 | if (!userfaultfd_is_initialized(ctx))
|
| 3626 | return -EINVAL;
|
| 3627 |
|
| 3628 | no_wait = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT;
|
| 3629 | for (;;) {
|
| 3630 | if (iov_iter_count(to) < sizeof(msg))
|
| 3631 | return ret ? ret : -EINVAL;
|
| 3632 | _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
|
| 3633 | if (_ret < 0)
|
| 3634 | return ret ? ret : _ret;
|
| 3635 | _ret = !copy_to_iter_full(&msg, sizeof(msg), to);
|
| 3636 | if (_ret)
|
| 3637 | return ret ? ret : -EFAULT;
|
| 3638 | ret += sizeof(msg);
|
| 3639 | |
| 3640 | * Allow to read more than one fault at time but only
|
| 3641 | * block if waiting for the very first one.
|
| 3642 |
|
| 3643 | no_wait = true;
|
| 3644 | }
|
| 3645 | }
|
| 3646 |
|
| 3647 | static void __wake_userfault(struct userfaultfd_ctx *ctx,
|
| 3648 | struct userfaultfd_wake_range *range)
|
| 3649 | {
|
| 3650 | spin_lock_irq(&ctx->fault_pending_wqh.lock);
|
| 3651 |
|
| 3652 | if (waitqueue_active(&ctx->fault_pending_wqh))
|
| 3653 | __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
|
| 3654 | range);
|
| 3655 | if (waitqueue_active(&ctx->fault_wqh))
|
| 3656 | __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
|
| 3657 | spin_unlock_irq(&ctx->fault_pending_wqh.lock);
|
| 3658 | }
|
| 3659 |
|
| 3660 | static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
|
| 3661 | struct userfaultfd_wake_range *range)
|
| 3662 | {
|
| 3663 | unsigned seq;
|
| 3664 | bool need_wakeup;
|
| 3665 |
|
| 3666 | |
| 3667 | * To be sure waitqueue_active() is not reordered by the CPU
|
| 3668 | * before the pagetable update, use an explicit SMP memory
|
| 3669 | * barrier here. PT lock release or mmap_read_unlock(mm) still
|
| 3670 | * have release semantics that can allow the
|
| 3671 | * waitqueue_active() to be reordered before the pte update.
|
| 3672 |
|
| 3673 | smp_mb();
|
| 3674 |
|
| 3675 | |
| 3676 | * Use waitqueue_active because it's very frequent to
|
| 3677 | * change the address space atomically even if there are no
|
| 3678 | * userfaults yet. So we take the spinlock only when we're
|
| 3679 | * sure we've userfaults to wake.
|
| 3680 |
|
| 3681 | do {
|
| 3682 | seq = read_seqcount_begin(&ctx->refile_seq);
|
| 3683 | need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
|
| 3684 | waitqueue_active(&ctx->fault_wqh);
|
| 3685 | cond_resched();
|
| 3686 | } while (read_seqcount_retry(&ctx->refile_seq, seq));
|
| 3687 | if (need_wakeup)
|
| 3688 | __wake_userfault(ctx, range);
|
| 3689 | }
|
| 3690 |
|
| 3691 | static __always_inline int validate_unaligned_range(
|
| 3692 | struct mm_struct *mm, __u64 start, __u64 len)
|
| 3693 | {
|
| 3694 | __u64 task_size = mm->task_size;
|
| 3695 |
|
| 3696 | if (len & ~PAGE_MASK)
|
| 3697 | return -EINVAL;
|
| 3698 | if (!len)
|
| 3699 | return -EINVAL;
|
| 3700 | if (start >= task_size)
|
| 3701 | return -EINVAL;
|
| 3702 | if (len > task_size - start)
|
| 3703 | return -EINVAL;
|
| 3704 | if (start + len <= start)
|
| 3705 | return -EINVAL;
|
| 3706 | return 0;
|
| 3707 | }
|
| 3708 |
|
| 3709 | static __always_inline int validate_range(struct mm_struct *mm,
|
| 3710 | __u64 start, __u64 len)
|
| 3711 | {
|
| 3712 | if (start & ~PAGE_MASK)
|
| 3713 | return -EINVAL;
|
| 3714 |
|
| 3715 | return validate_unaligned_range(mm, start, len);
|
| 3716 | }
|
| 3717 |
|
| 3718 | static int userfaultfd_register(struct userfaultfd_ctx *ctx,
|
| 3719 | unsigned long arg)
|
| 3720 | {
|
| 3721 | struct mm_struct *mm = ctx->mm;
|
| 3722 | struct vm_area_struct *vma, *cur;
|
| 3723 | int ret;
|
| 3724 | struct uffdio_register uffdio_register;
|
| 3725 | struct uffdio_register __user *user_uffdio_register;
|
| 3726 | vm_flags_t vm_flags;
|
| 3727 | bool found;
|
| 3728 | bool basic_ioctls;
|
| 3729 | unsigned long start, end;
|
| 3730 | struct vma_iterator vmi;
|
| 3731 | bool wp_async = userfaultfd_wp_async_ctx(ctx);
|
| 3732 |
|
| 3733 | user_uffdio_register = (struct uffdio_register __user *) arg;
|
| 3734 |
|
| 3735 | ret = -EFAULT;
|
| 3736 | if (copy_from_user(&uffdio_register, user_uffdio_register,
|
| 3737 | sizeof(uffdio_register)-sizeof(__u64)))
|
| 3738 | goto out;
|
| 3739 |
|
| 3740 | ret = -EINVAL;
|
| 3741 | if (!uffdio_register.mode)
|
| 3742 | goto out;
|
| 3743 | if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
|
| 3744 | goto out;
|
| 3745 | vm_flags = 0;
|
| 3746 | if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
|
| 3747 | vm_flags |= VM_UFFD_MISSING;
|
| 3748 | if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
|
| 3749 | if (!pgtable_supports_uffd())
|
| 3750 | goto out;
|
| 3751 |
|
| 3752 | vm_flags |= VM_UFFD_WP;
|
| 3753 | }
|
| 3754 | if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) {
|
| 3755 | if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE)
|
| 3756 | goto out;
|
| 3757 | if (!(userfaultfd_features(ctx) & UFFD_FEATURE_RWP))
|
| 3758 | goto out;
|
| 3759 | vm_flags |= VM_UFFD_RWP;
|
| 3760 | }
|
| 3761 |
|
| 3762 | |
| 3763 | * WP and RWP share the uffd PTE bit and
|
| 3764 | * cannot coexist in the same VMA — the bit would carry ambiguous
|
| 3765 | * semantics. Reject the combination up front.
|
| 3766 |
|
| 3767 | if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP))
|
| 3768 | goto out;
|
| 3769 |
|
| 3770 | if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
|
| 3771 | #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
|
| 3772 | goto out;
|
| 3773 | #endif
|
| 3774 | vm_flags |= VM_UFFD_MINOR;
|
| 3775 | }
|
| 3776 |
|
| 3777 | ret = validate_range(mm, uffdio_register.range.start,
|
| 3778 | uffdio_register.range.len);
|
| 3779 | if (ret)
|
| 3780 | goto out;
|
| 3781 |
|
| 3782 | start = uffdio_register.range.start;
|
| 3783 | end = start + uffdio_register.range.len;
|
| 3784 |
|
| 3785 | ret = -ENOMEM;
|
| 3786 | if (!mmget_not_zero(mm))
|
| 3787 | goto out;
|
| 3788 |
|
| 3789 | ret = -EINVAL;
|
| 3790 | mmap_write_lock(mm);
|
| 3791 | vma_iter_init(&vmi, mm, start);
|
| 3792 | vma = vma_find(&vmi, end);
|
| 3793 | if (!vma)
|
| 3794 | goto out_unlock;
|
| 3795 |
|
| 3796 | |
| 3797 | * If the first vma contains huge pages, make sure start address
|
| 3798 | * is aligned to huge page size.
|
| 3799 |
|
| 3800 | if (is_vm_hugetlb_page(vma)) {
|
| 3801 | unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
|
| 3802 |
|
| 3803 | if (start & (vma_hpagesize - 1))
|
| 3804 | goto out_unlock;
|
| 3805 | }
|
| 3806 |
|
| 3807 | |
| 3808 | * Search for not compatible vmas.
|
| 3809 |
|
| 3810 | found = false;
|
| 3811 | basic_ioctls = false;
|
| 3812 | cur = vma;
|
| 3813 | do {
|
| 3814 | cond_resched();
|
| 3815 |
|
| 3816 | VM_WARN_ON_ONCE(!!cur->vm_userfaultfd_ctx.ctx ^
|
| 3817 | !!(cur->vm_flags & __VM_UFFD_FLAGS));
|
| 3818 |
|
| 3819 |
|
| 3820 | ret = -EINVAL;
|
| 3821 | if (!vma_can_userfault(cur, vm_flags, wp_async))
|
| 3822 | goto out_unlock;
|
| 3823 |
|
| 3824 | |
| 3825 | * RWP uses protnone as an access-tracking marker. PROT_NONE
|
| 3826 | * VMAs have vm_page_prot == PAGE_NONE, so RWP resolution
|
| 3827 | * cannot make a page accessible again. Reject at register
|
| 3828 | * time only: a VMA that later becomes inaccessible via
|
| 3829 | * mprotect() must still be unregisterable, so this is not
|
| 3830 | * part of vma_can_userfault().
|
| 3831 |
|
| 3832 | if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur))
|
| 3833 | goto out_unlock;
|
| 3834 |
|
| 3835 | |
| 3836 | * UFFDIO_COPY will fill file holes even without
|
| 3837 | * PROT_WRITE. This check enforces that if this is a
|
| 3838 | * MAP_SHARED, the process has write permission to the backing
|
| 3839 | * file. If VM_MAYWRITE is set it also enforces that on a
|
| 3840 | * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
|
| 3841 | * F_WRITE_SEAL can be taken until the vma is destroyed.
|
| 3842 |
|
| 3843 | ret = -EPERM;
|
| 3844 | if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
|
| 3845 | goto out_unlock;
|
| 3846 |
|
| 3847 | |
| 3848 | * If this vma contains ending address, and huge pages
|
| 3849 | * check alignment.
|
| 3850 |
|
| 3851 | if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
|
| 3852 | end > cur->vm_start) {
|
| 3853 | unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
|
| 3854 |
|
| 3855 | ret = -EINVAL;
|
| 3856 |
|
| 3857 | if (end & (vma_hpagesize - 1))
|
| 3858 | goto out_unlock;
|
| 3859 | }
|
| 3860 | if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
|
| 3861 | goto out_unlock;
|
| 3862 |
|
| 3863 | |
| 3864 | * Check that this vma isn't already owned by a
|
| 3865 | * different userfaultfd. We can't allow more than one
|
| 3866 | * userfaultfd to own a single vma simultaneously or we
|
| 3867 | * wouldn't know which one to deliver the userfaults to.
|
| 3868 |
|
| 3869 | ret = -EBUSY;
|
| 3870 | if (cur->vm_userfaultfd_ctx.ctx &&
|
| 3871 | cur->vm_userfaultfd_ctx.ctx != ctx)
|
| 3872 | goto out_unlock;
|
| 3873 |
|
| 3874 | |
| 3875 | * Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would
|
| 3876 | * leave PTE markers without the flag that describes them;
|
| 3877 | * subsequent mprotect() would then promote stale markers
|
| 3878 | * into the other mode. Require an unregister first.
|
| 3879 |
|
| 3880 | if (cur->vm_userfaultfd_ctx.ctx == ctx &&
|
| 3881 | cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags)
|
| 3882 | goto out_unlock;
|
| 3883 |
|
| 3884 | |
| 3885 | * Note vmas containing huge pages
|
| 3886 |
|
| 3887 | if (is_vm_hugetlb_page(cur))
|
| 3888 | basic_ioctls = true;
|
| 3889 |
|
| 3890 | found = true;
|
| 3891 | } for_each_vma_range(vmi, cur, end);
|
| 3892 | VM_WARN_ON_ONCE(!found);
|
| 3893 |
|
| 3894 | ret = userfaultfd_register_range(ctx, vma, vm_flags, start, end,
|
| 3895 | wp_async);
|
| 3896 |
|
| 3897 | out_unlock:
|
| 3898 | mmap_write_unlock(mm);
|
| 3899 | mmput(mm);
|
| 3900 | if (!ret) {
|
| 3901 | __u64 ioctls_out;
|
| 3902 |
|
| 3903 | ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
|
| 3904 | UFFD_API_RANGE_IOCTLS;
|
| 3905 |
|
| 3906 | |
| 3907 | * Declare the WP ioctl only if the WP mode is
|
| 3908 | * specified and all checks passed with the range
|
| 3909 |
|
| 3910 | if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
|
| 3911 | ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
|
| 3912 |
|
| 3913 |
|
| 3914 | if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
|
| 3915 | ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
|
| 3916 |
|
| 3917 |
|
| 3918 | if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP))
|
| 3919 | ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT);
|
| 3920 |
|
| 3921 | |
| 3922 | * Now that we scanned all vmas we can already tell
|
| 3923 | * userland which ioctls methods are guaranteed to
|
| 3924 | * succeed on this range.
|
| 3925 |
|
| 3926 | if (put_user(ioctls_out, &user_uffdio_register->ioctls))
|
| 3927 | ret = -EFAULT;
|
| 3928 | }
|
| 3929 | out:
|
| 3930 | return ret;
|
| 3931 | }
|
| 3932 |
|
| 3933 | static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
|
| 3934 | unsigned long arg)
|
| 3935 | {
|
| 3936 | struct mm_struct *mm = ctx->mm;
|
| 3937 | struct vm_area_struct *vma, *prev, *cur;
|
| 3938 | int ret;
|
| 3939 | struct uffdio_range uffdio_unregister;
|
| 3940 | bool found;
|
| 3941 | unsigned long start, end, vma_end;
|
| 3942 | const void __user *buf = (void __user *)arg;
|
| 3943 | struct vma_iterator vmi;
|
| 3944 | bool wp_async = userfaultfd_wp_async_ctx(ctx);
|
| 3945 |
|
| 3946 | ret = -EFAULT;
|
| 3947 | if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
|
| 3948 | goto out;
|
| 3949 |
|
| 3950 | ret = validate_range(mm, uffdio_unregister.start,
|
| 3951 | uffdio_unregister.len);
|
| 3952 | if (ret)
|
| 3953 | goto out;
|
| 3954 |
|
| 3955 | start = uffdio_unregister.start;
|
| 3956 | end = start + uffdio_unregister.len;
|
| 3957 |
|
| 3958 | ret = -ENOMEM;
|
| 3959 | if (!mmget_not_zero(mm))
|
| 3960 | goto out;
|
| 3961 |
|
| 3962 | mmap_write_lock(mm);
|
| 3963 | ret = -EINVAL;
|
| 3964 | vma_iter_init(&vmi, mm, start);
|
| 3965 | vma = vma_find(&vmi, end);
|
| 3966 | if (!vma)
|
| 3967 | goto out_unlock;
|
| 3968 |
|
| 3969 | |
| 3970 | * If the first vma contains huge pages, make sure start address
|
| 3971 | * is aligned to huge page size.
|
| 3972 |
|
| 3973 | if (is_vm_hugetlb_page(vma)) {
|
| 3974 | unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
|
| 3975 |
|
| 3976 | if (start & (vma_hpagesize - 1))
|
| 3977 | goto out_unlock;
|
| 3978 | }
|
| 3979 |
|
| 3980 | |
| 3981 | * Search for not compatible vmas.
|
| 3982 |
|
| 3983 | found = false;
|
| 3984 | cur = vma;
|
| 3985 | do {
|
| 3986 | cond_resched();
|
| 3987 |
|
| 3988 | VM_WARN_ON_ONCE(!!cur->vm_userfaultfd_ctx.ctx ^
|
| 3989 | !!(cur->vm_flags & __VM_UFFD_FLAGS));
|
| 3990 |
|
| 3991 | |
| 3992 | * Prevent unregistering through a different userfaultfd than
|
| 3993 | * the one used for registration.
|
| 3994 |
|
| 3995 | if (cur->vm_userfaultfd_ctx.ctx &&
|
| 3996 | cur->vm_userfaultfd_ctx.ctx != ctx)
|
| 3997 | goto out_unlock;
|
| 3998 |
|
| 3999 | |
| 4000 | * Check not compatible vmas, not strictly required
|
| 4001 | * here as not compatible vmas cannot have an
|
| 4002 | * userfaultfd_ctx registered on them, but this
|
| 4003 | * provides for more strict behavior to notice
|
| 4004 | * unregistration errors.
|
| 4005 |
|
| 4006 | if (!vma_can_userfault(cur, cur->vm_flags, wp_async))
|
| 4007 | goto out_unlock;
|
| 4008 |
|
| 4009 | found = true;
|
| 4010 | } for_each_vma_range(vmi, cur, end);
|
| 4011 | VM_WARN_ON_ONCE(!found);
|
| 4012 |
|
| 4013 | vma_iter_set(&vmi, start);
|
| 4014 | prev = vma_prev(&vmi);
|
| 4015 | if (vma->vm_start < start)
|
| 4016 | prev = vma;
|
| 4017 |
|
| 4018 | ret = 0;
|
| 4019 | for_each_vma_range(vmi, vma, end) {
|
| 4020 | cond_resched();
|
| 4021 |
|
| 4022 |
|
| 4023 | if (!vma->vm_userfaultfd_ctx.ctx)
|
| 4024 | goto skip;
|
| 4025 |
|
| 4026 | VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx != ctx);
|
| 4027 | VM_WARN_ON_ONCE(!vma_can_userfault(vma, vma->vm_flags, wp_async));
|
| 4028 | VM_WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE));
|
| 4029 |
|
| 4030 | if (vma->vm_start > start)
|
| 4031 | start = vma->vm_start;
|
| 4032 | vma_end = min(end, vma->vm_end);
|
| 4033 |
|
| 4034 | if (userfaultfd_missing(vma)) {
|
| 4035 | |
| 4036 | * Wake any concurrent pending userfault while
|
| 4037 | * we unregister, so they will not hang
|
| 4038 | * permanently and it avoids userland to call
|
| 4039 | * UFFDIO_WAKE explicitly.
|
| 4040 |
|
| 4041 | struct userfaultfd_wake_range range;
|
| 4042 | range.start = start;
|
| 4043 | range.len = vma_end - start;
|
| 4044 | wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
|
| 4045 | }
|
| 4046 |
|
| 4047 | vma = userfaultfd_clear_vma(&vmi, prev, vma,
|
| 4048 | start, vma_end);
|
| 4049 | if (IS_ERR(vma)) {
|
| 4050 | ret = PTR_ERR(vma);
|
| 4051 | break;
|
| 4052 | }
|
| 4053 |
|
| 4054 | skip:
|
| 4055 | prev = vma;
|
| 4056 | start = vma->vm_end;
|
| 4057 | }
|
| 4058 |
|
| 4059 | out_unlock:
|
| 4060 | mmap_write_unlock(mm);
|
| 4061 | mmput(mm);
|
| 4062 | out:
|
| 4063 | return ret;
|
| 4064 | }
|
| 4065 |
|
| 4066 | |
| 4067 | * userfaultfd_wake may be used in combination with the
|
| 4068 | * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
|
| 4069 |
|
| 4070 | static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
|
| 4071 | unsigned long arg)
|
| 4072 | {
|
| 4073 | int ret;
|
| 4074 | struct uffdio_range uffdio_wake;
|
| 4075 | struct userfaultfd_wake_range range;
|
| 4076 | const void __user *buf = (void __user *)arg;
|
| 4077 |
|
| 4078 | ret = -EFAULT;
|
| 4079 | if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
|
| 4080 | goto out;
|
| 4081 |
|
| 4082 | ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
|
| 4083 | if (ret)
|
| 4084 | goto out;
|
| 4085 |
|
| 4086 | range.start = uffdio_wake.start;
|
| 4087 | range.len = uffdio_wake.len;
|
| 4088 |
|
| 4089 | |
| 4090 | * len == 0 means wake all and we don't want to wake all here,
|
| 4091 | * so check it again to be sure.
|
| 4092 |
|
| 4093 | VM_WARN_ON_ONCE(!range.len);
|
| 4094 |
|
| 4095 | wake_userfault(ctx, &range);
|
| 4096 | ret = 0;
|
| 4097 |
|
| 4098 | out:
|
| 4099 | return ret;
|
| 4100 | }
|
| 4101 |
|
| 4102 | static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
|
| 4103 | unsigned long arg)
|
| 4104 | {
|
| 4105 | __s64 ret;
|
| 4106 | struct uffdio_copy uffdio_copy;
|
| 4107 | struct uffdio_copy __user *user_uffdio_copy;
|
| 4108 | struct userfaultfd_wake_range range;
|
| 4109 | uffd_flags_t flags = 0;
|
| 4110 |
|
| 4111 | user_uffdio_copy = (struct uffdio_copy __user *) arg;
|
| 4112 |
|
| 4113 | ret = -EAGAIN;
|
| 4114 | if (unlikely(atomic_read(&ctx->mmap_changing))) {
|
| 4115 | if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
|
| 4116 | return -EFAULT;
|
| 4117 | goto out;
|
| 4118 | }
|
| 4119 |
|
| 4120 | ret = -EFAULT;
|
| 4121 | if (copy_from_user(&uffdio_copy, user_uffdio_copy,
|
| 4122 |
|
| 4123 | sizeof(uffdio_copy)-sizeof(__s64)))
|
| 4124 | goto out;
|
| 4125 |
|
| 4126 | ret = validate_unaligned_range(ctx->mm, uffdio_copy.src,
|
| 4127 | uffdio_copy.len);
|
| 4128 | if (ret)
|
| 4129 | goto out;
|
| 4130 | ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
|
| 4131 | if (ret)
|
| 4132 | goto out;
|
| 4133 |
|
| 4134 | ret = -EINVAL;
|
| 4135 | if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
|
| 4136 | goto out;
|
| 4137 | if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP)
|
| 4138 | flags |= MFILL_ATOMIC_WP;
|
| 4139 | if (mmget_not_zero(ctx->mm)) {
|
| 4140 | ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src,
|
| 4141 | uffdio_copy.len, flags);
|
| 4142 | mmput(ctx->mm);
|
| 4143 | } else {
|
| 4144 | return -ESRCH;
|
| 4145 | }
|
| 4146 | if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
|
| 4147 | return -EFAULT;
|
| 4148 | if (ret < 0)
|
| 4149 | goto out;
|
| 4150 | VM_WARN_ON_ONCE(!ret);
|
| 4151 |
|
| 4152 | range.len = ret;
|
| 4153 | if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
|
| 4154 | range.start = uffdio_copy.dst;
|
| 4155 | wake_userfault(ctx, &range);
|
| 4156 | }
|
| 4157 | ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
|
| 4158 | out:
|
| 4159 | return ret;
|
| 4160 | }
|
| 4161 |
|
| 4162 | static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
|
| 4163 | unsigned long arg)
|
| 4164 | {
|
| 4165 | __s64 ret;
|
| 4166 | struct uffdio_zeropage uffdio_zeropage;
|
| 4167 | struct uffdio_zeropage __user *user_uffdio_zeropage;
|
| 4168 | struct userfaultfd_wake_range range;
|
| 4169 |
|
| 4170 | user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
|
| 4171 |
|
| 4172 | ret = -EAGAIN;
|
| 4173 | if (unlikely(atomic_read(&ctx->mmap_changing))) {
|
| 4174 | if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
|
| 4175 | return -EFAULT;
|
| 4176 | goto out;
|
| 4177 | }
|
| 4178 |
|
| 4179 | ret = -EFAULT;
|
| 4180 | if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
|
| 4181 |
|
| 4182 | sizeof(uffdio_zeropage)-sizeof(__s64)))
|
| 4183 | goto out;
|
| 4184 |
|
| 4185 | ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
|
| 4186 | uffdio_zeropage.range.len);
|
| 4187 | if (ret)
|
| 4188 | goto out;
|
| 4189 | ret = -EINVAL;
|
| 4190 | if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
|
| 4191 | goto out;
|
| 4192 |
|
| 4193 | if (mmget_not_zero(ctx->mm)) {
|
| 4194 | ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start,
|
| 4195 | uffdio_zeropage.range.len);
|
| 4196 | mmput(ctx->mm);
|
| 4197 | } else {
|
| 4198 | return -ESRCH;
|
| 4199 | }
|
| 4200 | if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
|
| 4201 | return -EFAULT;
|
| 4202 | if (ret < 0)
|
| 4203 | goto out;
|
| 4204 |
|
| 4205 | VM_WARN_ON_ONCE(!ret);
|
| 4206 | range.len = ret;
|
| 4207 | if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
|
| 4208 | range.start = uffdio_zeropage.range.start;
|
| 4209 | wake_userfault(ctx, &range);
|
| 4210 | }
|
| 4211 | ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
|
| 4212 | out:
|
| 4213 | return ret;
|
| 4214 | }
|
| 4215 |
|
| 4216 | static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
|
| 4217 | unsigned long arg)
|
| 4218 | {
|
| 4219 | int ret;
|
| 4220 | struct uffdio_writeprotect uffdio_wp;
|
| 4221 | struct uffdio_writeprotect __user *user_uffdio_wp;
|
| 4222 | struct userfaultfd_wake_range range;
|
| 4223 | bool mode_wp, mode_dontwake;
|
| 4224 |
|
| 4225 | if (atomic_read(&ctx->mmap_changing))
|
| 4226 | return -EAGAIN;
|
| 4227 |
|
| 4228 | user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
|
| 4229 |
|
| 4230 | if (copy_from_user(&uffdio_wp, user_uffdio_wp,
|
| 4231 | sizeof(struct uffdio_writeprotect)))
|
| 4232 | return -EFAULT;
|
| 4233 |
|
| 4234 | ret = validate_range(ctx->mm, uffdio_wp.range.start,
|
| 4235 | uffdio_wp.range.len);
|
| 4236 | if (ret)
|
| 4237 | return ret;
|
| 4238 |
|
| 4239 | if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
|
| 4240 | UFFDIO_WRITEPROTECT_MODE_WP))
|
| 4241 | return -EINVAL;
|
| 4242 |
|
| 4243 | mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
|
| 4244 | mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
|
| 4245 |
|
| 4246 | if (mode_wp && mode_dontwake)
|
| 4247 | return -EINVAL;
|
| 4248 |
|
| 4249 | if (mmget_not_zero(ctx->mm)) {
|
| 4250 | ret = mwriteprotect_range(ctx, uffdio_wp.range.start,
|
| 4251 | uffdio_wp.range.len, mode_wp);
|
| 4252 | mmput(ctx->mm);
|
| 4253 | } else {
|
| 4254 | return -ESRCH;
|
| 4255 | }
|
| 4256 |
|
| 4257 | if (ret)
|
| 4258 | return ret;
|
| 4259 |
|
| 4260 | if (!mode_wp && !mode_dontwake) {
|
| 4261 | range.start = uffdio_wp.range.start;
|
| 4262 | range.len = uffdio_wp.range.len;
|
| 4263 | wake_userfault(ctx, &range);
|
| 4264 | }
|
| 4265 | return ret;
|
| 4266 | }
|
| 4267 |
|
| 4268 | static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx,
|
| 4269 | unsigned long arg)
|
| 4270 | {
|
| 4271 | int ret;
|
| 4272 | struct uffdio_rwprotect uffdio_rwp;
|
| 4273 | struct userfaultfd_wake_range range;
|
| 4274 | bool mode_rwp, mode_dontwake;
|
| 4275 |
|
| 4276 | if (atomic_read(&ctx->mmap_changing))
|
| 4277 | return -EAGAIN;
|
| 4278 |
|
| 4279 | if (copy_from_user(&uffdio_rwp, (void __user *)arg,
|
| 4280 | sizeof(uffdio_rwp)))
|
| 4281 | return -EFAULT;
|
| 4282 |
|
| 4283 | ret = validate_range(ctx->mm, uffdio_rwp.range.start,
|
| 4284 | uffdio_rwp.range.len);
|
| 4285 | if (ret)
|
| 4286 | return ret;
|
| 4287 |
|
| 4288 | if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE |
|
| 4289 | UFFDIO_RWPROTECT_MODE_RWP))
|
| 4290 | return -EINVAL;
|
| 4291 |
|
| 4292 | mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP;
|
| 4293 | mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE;
|
| 4294 |
|
| 4295 | if (mode_rwp && mode_dontwake)
|
| 4296 | return -EINVAL;
|
| 4297 |
|
| 4298 | if (mmget_not_zero(ctx->mm)) {
|
| 4299 | ret = mrwprotect_range(ctx, uffdio_rwp.range.start,
|
| 4300 | uffdio_rwp.range.len, mode_rwp);
|
| 4301 | mmput(ctx->mm);
|
| 4302 | } else {
|
| 4303 | return -ESRCH;
|
| 4304 | }
|
| 4305 |
|
| 4306 | if (ret)
|
| 4307 | return ret;
|
| 4308 |
|
| 4309 | if (!mode_rwp && !mode_dontwake) {
|
| 4310 | range.start = uffdio_rwp.range.start;
|
| 4311 | range.len = uffdio_rwp.range.len;
|
| 4312 | wake_userfault(ctx, &range);
|
| 4313 | }
|
| 4314 | return ret;
|
| 4315 | }
|
| 4316 |
|
| 4317 |
|
| 4318 | static __u64 uffd_api_available_features(void)
|
| 4319 | {
|
| 4320 | __u64 f = UFFD_API_FEATURES;
|
| 4321 |
|
| 4322 | if (!IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_MINOR))
|
| 4323 | f &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
|
| 4324 | if (!pgtable_supports_uffd())
|
| 4325 | f &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
|
| 4326 | if (!uffd_supports_wp_marker())
|
| 4327 | f &= ~(UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
|
| 4328 | UFFD_FEATURE_WP_UNPOPULATED |
|
| 4329 | UFFD_FEATURE_WP_ASYNC);
|
| 4330 | |
| 4331 | * RWP needs both PROT_NONE support and the uffd PTE bit. The
|
| 4332 | * VM_UFFD_RWP check covers compile-time unavailability; the
|
| 4333 | * pgtable_supports_uffd() check covers runtime (e.g. riscv
|
| 4334 | * without the SVRSW60T59B extension) where the PTE bit is declared
|
| 4335 | * but not actually usable.
|
| 4336 |
|
| 4337 | if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd())
|
| 4338 | f &= ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC);
|
| 4339 | return f;
|
| 4340 | }
|
| 4341 |
|
| 4342 |
|
| 4343 | #define UFFD_FEATURE_TOGGLEABLE UFFD_FEATURE_RWP_ASYNC
|
| 4344 |
|
| 4345 | static int userfaultfd_set_mode(struct userfaultfd_ctx *ctx,
|
| 4346 | unsigned long arg)
|
| 4347 | {
|
| 4348 | struct uffdio_set_mode mode;
|
| 4349 | struct mm_struct *mm = ctx->mm;
|
| 4350 |
|
| 4351 | if (copy_from_user(&mode, (void __user *)arg, sizeof(mode)))
|
| 4352 | return -EFAULT;
|
| 4353 |
|
| 4354 |
|
| 4355 | if (mode.enable & mode.disable)
|
| 4356 | return -EINVAL;
|
| 4357 |
|
| 4358 |
|
| 4359 | if ((mode.enable | mode.disable) &
|
| 4360 | ~(uffd_api_available_features() & UFFD_FEATURE_TOGGLEABLE))
|
| 4361 | return -EINVAL;
|
| 4362 |
|
| 4363 |
|
| 4364 | if ((mode.enable & UFFD_FEATURE_RWP_ASYNC) &&
|
| 4365 | !(userfaultfd_features(ctx) & UFFD_FEATURE_RWP))
|
| 4366 | return -EINVAL;
|
| 4367 |
|
| 4368 | if (!mmget_not_zero(mm))
|
| 4369 | return -ESRCH;
|
| 4370 |
|
| 4371 | |
| 4372 | * Drain in-flight faults before flipping features. mmap_write_lock()
|
| 4373 | * blocks new mmap_read_lock() callers, but per-VMA locked faults
|
| 4374 | * (lock_vma_under_rcu() + FAULT_FLAG_VMA_LOCK) that acquired before
|
| 4375 | * this point keep running. Calling vma_start_write() on each UFFD-
|
| 4376 | * armed VMA waits for those readers to drop, so no in-flight fault
|
| 4377 | * can observe the old features after mmap_write_unlock().
|
| 4378 |
|
| 4379 | mmap_write_lock(mm);
|
| 4380 | {
|
| 4381 | struct vm_area_struct *vma;
|
| 4382 | VMA_ITERATOR(vmi, mm, 0);
|
| 4383 |
|
| 4384 | for_each_vma(vmi, vma) {
|
| 4385 | if (vma->vm_userfaultfd_ctx.ctx == ctx)
|
| 4386 | vma_start_write(vma);
|
| 4387 | }
|
| 4388 | }
|
| 4389 | |
| 4390 | * Single WRITE_ONCE so lockless readers (fdinfo, poll/read_iter
|
| 4391 | * via userfaultfd_is_initialized(), and the userfaultfd_features()
|
| 4392 | * helper used elsewhere) can't observe a mid-RMW intermediate
|
| 4393 | * value. Hot-path readers already serialise through the mmap lock
|
| 4394 | * + vma_start_write() drain above, so their load doesn't need an
|
| 4395 | * annotation.
|
| 4396 |
|
| 4397 | WRITE_ONCE(ctx->features,
|
| 4398 | (ctx->features | mode.enable) & ~mode.disable);
|
| 4399 | mmap_write_unlock(mm);
|
| 4400 |
|
| 4401 | |
| 4402 | * If switching to async, wake threads blocked in handle_userfault().
|
| 4403 | * They will retry the fault and auto-resolve under the new mode.
|
| 4404 | * len=0 means wake all pending faults on this context.
|
| 4405 |
|
| 4406 | if (mode.enable & UFFD_FEATURE_RWP_ASYNC) {
|
| 4407 | struct userfaultfd_wake_range range = { .len = 0 };
|
| 4408 |
|
| 4409 | spin_lock_irq(&ctx->fault_pending_wqh.lock);
|
| 4410 | __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
|
| 4411 | &range);
|
| 4412 | __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
|
| 4413 | spin_unlock_irq(&ctx->fault_pending_wqh.lock);
|
| 4414 | }
|
| 4415 |
|
| 4416 | mmput(mm);
|
| 4417 | return 0;
|
| 4418 | }
|
| 4419 |
|
| 4420 | static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
|
| 4421 | {
|
| 4422 | __s64 ret;
|
| 4423 | struct uffdio_continue uffdio_continue;
|
| 4424 | struct uffdio_continue __user *user_uffdio_continue;
|
| 4425 | struct userfaultfd_wake_range range;
|
| 4426 | uffd_flags_t flags = 0;
|
| 4427 |
|
| 4428 | user_uffdio_continue = (struct uffdio_continue __user *)arg;
|
| 4429 |
|
| 4430 | ret = -EAGAIN;
|
| 4431 | if (unlikely(atomic_read(&ctx->mmap_changing))) {
|
| 4432 | if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
|
| 4433 | return -EFAULT;
|
| 4434 | goto out;
|
| 4435 | }
|
| 4436 |
|
| 4437 | ret = -EFAULT;
|
| 4438 | if (copy_from_user(&uffdio_continue, user_uffdio_continue,
|
| 4439 |
|
| 4440 | sizeof(uffdio_continue) - (sizeof(__s64))))
|
| 4441 | goto out;
|
| 4442 |
|
| 4443 | ret = validate_range(ctx->mm, uffdio_continue.range.start,
|
| 4444 | uffdio_continue.range.len);
|
| 4445 | if (ret)
|
| 4446 | goto out;
|
| 4447 |
|
| 4448 | ret = -EINVAL;
|
| 4449 | if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE |
|
| 4450 | UFFDIO_CONTINUE_MODE_WP))
|
| 4451 | goto out;
|
| 4452 | if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP)
|
| 4453 | flags |= MFILL_ATOMIC_WP;
|
| 4454 |
|
| 4455 | if (mmget_not_zero(ctx->mm)) {
|
| 4456 | ret = mfill_atomic_continue(ctx, uffdio_continue.range.start,
|
| 4457 | uffdio_continue.range.len, flags);
|
| 4458 | mmput(ctx->mm);
|
| 4459 | } else {
|
| 4460 | return -ESRCH;
|
| 4461 | }
|
| 4462 |
|
| 4463 | if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
|
| 4464 | return -EFAULT;
|
| 4465 | if (ret < 0)
|
| 4466 | goto out;
|
| 4467 |
|
| 4468 |
|
| 4469 | VM_WARN_ON_ONCE(!ret);
|
| 4470 | range.len = ret;
|
| 4471 | if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
|
| 4472 | range.start = uffdio_continue.range.start;
|
| 4473 | wake_userfault(ctx, &range);
|
| 4474 | }
|
| 4475 | ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
|
| 4476 |
|
| 4477 | out:
|
| 4478 | return ret;
|
| 4479 | }
|
| 4480 |
|
| 4481 | static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg)
|
| 4482 | {
|
| 4483 | __s64 ret;
|
| 4484 | struct uffdio_poison uffdio_poison;
|
| 4485 | struct uffdio_poison __user *user_uffdio_poison;
|
| 4486 | struct userfaultfd_wake_range range;
|
| 4487 |
|
| 4488 | user_uffdio_poison = (struct uffdio_poison __user *)arg;
|
| 4489 |
|
| 4490 | ret = -EAGAIN;
|
| 4491 | if (unlikely(atomic_read(&ctx->mmap_changing))) {
|
| 4492 | if (unlikely(put_user(ret, &user_uffdio_poison->updated)))
|
| 4493 | return -EFAULT;
|
| 4494 | goto out;
|
| 4495 | }
|
| 4496 |
|
| 4497 | ret = -EFAULT;
|
| 4498 | if (copy_from_user(&uffdio_poison, user_uffdio_poison,
|
| 4499 |
|
| 4500 | sizeof(uffdio_poison) - (sizeof(__s64))))
|
| 4501 | goto out;
|
| 4502 |
|
| 4503 | ret = validate_range(ctx->mm, uffdio_poison.range.start,
|
| 4504 | uffdio_poison.range.len);
|
| 4505 | if (ret)
|
| 4506 | goto out;
|
| 4507 |
|
| 4508 | ret = -EINVAL;
|
| 4509 | if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE)
|
| 4510 | goto out;
|
| 4511 |
|
| 4512 | if (mmget_not_zero(ctx->mm)) {
|
| 4513 | ret = mfill_atomic_poison(ctx, uffdio_poison.range.start,
|
| 4514 | uffdio_poison.range.len, 0);
|
| 4515 | mmput(ctx->mm);
|
| 4516 | } else {
|
| 4517 | return -ESRCH;
|
| 4518 | }
|
| 4519 |
|
| 4520 | if (unlikely(put_user(ret, &user_uffdio_poison->updated)))
|
| 4521 | return -EFAULT;
|
| 4522 | if (ret < 0)
|
| 4523 | goto out;
|
| 4524 |
|
| 4525 |
|
| 4526 | VM_WARN_ON_ONCE(!ret);
|
| 4527 | range.len = ret;
|
| 4528 | if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) {
|
| 4529 | range.start = uffdio_poison.range.start;
|
| 4530 | wake_userfault(ctx, &range);
|
| 4531 | }
|
| 4532 | ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN;
|
| 4533 |
|
| 4534 | out:
|
| 4535 | return ret;
|
| 4536 | }
|
| 4537 |
|
| 4538 | bool userfaultfd_wp_async(struct vm_area_struct *vma)
|
| 4539 | {
|
| 4540 | return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx);
|
| 4541 | }
|
| 4542 |
|
| 4543 | bool userfaultfd_rwp_async(struct vm_area_struct *vma)
|
| 4544 | {
|
| 4545 | return userfaultfd_rwp_async_ctx(vma->vm_userfaultfd_ctx.ctx);
|
| 4546 | }
|
| 4547 |
|
| 4548 | static inline unsigned int uffd_ctx_features(__u64 user_features)
|
| 4549 | {
|
| 4550 | |
| 4551 | * For the current set of features the bits just coincide. Set
|
| 4552 | * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
|
| 4553 |
|
| 4554 | return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
|
| 4555 | }
|
| 4556 |
|
| 4557 | static int userfaultfd_move(struct userfaultfd_ctx *ctx,
|
| 4558 | unsigned long arg)
|
| 4559 | {
|
| 4560 | __s64 ret;
|
| 4561 | struct uffdio_move uffdio_move;
|
| 4562 | struct uffdio_move __user *user_uffdio_move;
|
| 4563 | struct userfaultfd_wake_range range;
|
| 4564 | struct mm_struct *mm = ctx->mm;
|
| 4565 |
|
| 4566 | user_uffdio_move = (struct uffdio_move __user *) arg;
|
| 4567 |
|
| 4568 | ret = -EAGAIN;
|
| 4569 | if (unlikely(atomic_read(&ctx->mmap_changing))) {
|
| 4570 | if (unlikely(put_user(ret, &user_uffdio_move->move)))
|
| 4571 | return -EFAULT;
|
| 4572 | goto out;
|
| 4573 | }
|
| 4574 |
|
| 4575 | if (copy_from_user(&uffdio_move, user_uffdio_move,
|
| 4576 |
|
| 4577 | sizeof(uffdio_move)-sizeof(__s64)))
|
| 4578 | return -EFAULT;
|
| 4579 |
|
| 4580 |
|
| 4581 | if (mm != current->mm)
|
| 4582 | return -EINVAL;
|
| 4583 |
|
| 4584 | ret = validate_range(mm, uffdio_move.dst, uffdio_move.len);
|
| 4585 | if (ret)
|
| 4586 | return ret;
|
| 4587 |
|
| 4588 | ret = validate_range(mm, uffdio_move.src, uffdio_move.len);
|
| 4589 | if (ret)
|
| 4590 | return ret;
|
| 4591 |
|
| 4592 | if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES|
|
| 4593 | UFFDIO_MOVE_MODE_DONTWAKE))
|
| 4594 | return -EINVAL;
|
| 4595 |
|
| 4596 | if (mmget_not_zero(mm)) {
|
| 4597 | ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src,
|
| 4598 | uffdio_move.len, uffdio_move.mode);
|
| 4599 | mmput(mm);
|
| 4600 | } else {
|
| 4601 | return -ESRCH;
|
| 4602 | }
|
| 4603 |
|
| 4604 | if (unlikely(put_user(ret, &user_uffdio_move->move)))
|
| 4605 | return -EFAULT;
|
| 4606 | if (ret < 0)
|
| 4607 | goto out;
|
| 4608 |
|
| 4609 |
|
| 4610 | VM_WARN_ON(!ret);
|
| 4611 | range.len = ret;
|
| 4612 | if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) {
|
| 4613 | range.start = uffdio_move.dst;
|
| 4614 | wake_userfault(ctx, &range);
|
| 4615 | }
|
| 4616 | ret = range.len == uffdio_move.len ? 0 : -EAGAIN;
|
| 4617 |
|
| 4618 | out:
|
| 4619 | return ret;
|
| 4620 | }
|
| 4621 |
|
| 4622 | |
| 4623 | * userland asks for a certain API version and we return which bits
|
| 4624 | * and ioctl commands are implemented in this kernel for such API
|
| 4625 | * version or -EINVAL if unknown.
|
| 4626 |
|
| 4627 | static int userfaultfd_api(struct userfaultfd_ctx *ctx,
|
| 4628 | unsigned long arg)
|
| 4629 | {
|
| 4630 | struct uffdio_api uffdio_api;
|
| 4631 | void __user *buf = (void __user *)arg;
|
| 4632 | unsigned int ctx_features;
|
| 4633 | int ret;
|
| 4634 | __u64 features;
|
| 4635 |
|
| 4636 | ret = -EFAULT;
|
| 4637 | if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
|
| 4638 | goto out;
|
| 4639 | features = uffdio_api.features;
|
| 4640 | ret = -EINVAL;
|
| 4641 | if (uffdio_api.api != UFFD_API)
|
| 4642 | goto err_out;
|
| 4643 | ret = -EPERM;
|
| 4644 | if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
|
| 4645 | goto err_out;
|
| 4646 |
|
| 4647 |
|
| 4648 | if (features & UFFD_FEATURE_WP_ASYNC)
|
| 4649 | features |= UFFD_FEATURE_WP_UNPOPULATED;
|
| 4650 |
|
| 4651 | ret = -EINVAL;
|
| 4652 |
|
| 4653 | if ((features & UFFD_FEATURE_RWP_ASYNC) &&
|
| 4654 | !(features & UFFD_FEATURE_RWP))
|
| 4655 | goto err_out;
|
| 4656 |
|
| 4657 |
|
| 4658 | uffdio_api.features = uffd_api_available_features();
|
| 4659 |
|
| 4660 | ret = -EINVAL;
|
| 4661 | if (features & ~uffdio_api.features)
|
| 4662 | goto err_out;
|
| 4663 |
|
| 4664 | uffdio_api.ioctls = UFFD_API_IOCTLS;
|
| 4665 | ret = -EFAULT;
|
| 4666 | if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
|
| 4667 | goto out;
|
| 4668 |
|
| 4669 |
|
| 4670 | ctx_features = uffd_ctx_features(features);
|
| 4671 | ret = -EINVAL;
|
| 4672 | if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
|
| 4673 | goto err_out;
|
| 4674 |
|
| 4675 | ret = 0;
|
| 4676 | out:
|
| 4677 | return ret;
|
| 4678 | err_out:
|
| 4679 | memset(&uffdio_api, 0, sizeof(uffdio_api));
|
| 4680 | if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
|
| 4681 | ret = -EFAULT;
|
| 4682 | goto out;
|
| 4683 | }
|
| 4684 |
|
| 4685 | static long userfaultfd_ioctl(struct file *file, unsigned cmd,
|
| 4686 | unsigned long arg)
|
| 4687 | {
|
| 4688 | int ret = -EINVAL;
|
| 4689 | struct userfaultfd_ctx *ctx = file->private_data;
|
| 4690 |
|
| 4691 | if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
|
| 4692 | return -EINVAL;
|
| 4693 |
|
| 4694 | switch (cmd) {
|
| 4695 | case UFFDIO_API:
|
| 4696 | ret = userfaultfd_api(ctx, arg);
|
| 4697 | break;
|
| 4698 | case UFFDIO_REGISTER:
|
| 4699 | ret = userfaultfd_register(ctx, arg);
|
| 4700 | break;
|
| 4701 | case UFFDIO_UNREGISTER:
|
| 4702 | ret = userfaultfd_unregister(ctx, arg);
|
| 4703 | break;
|
| 4704 | case UFFDIO_WAKE:
|
| 4705 | ret = userfaultfd_wake(ctx, arg);
|
| 4706 | break;
|
| 4707 | case UFFDIO_COPY:
|
| 4708 | ret = userfaultfd_copy(ctx, arg);
|
| 4709 | break;
|
| 4710 | case UFFDIO_ZEROPAGE:
|
| 4711 | ret = userfaultfd_zeropage(ctx, arg);
|
| 4712 | break;
|
| 4713 | case UFFDIO_MOVE:
|
| 4714 | ret = userfaultfd_move(ctx, arg);
|
| 4715 | break;
|
| 4716 | case UFFDIO_WRITEPROTECT:
|
| 4717 | ret = userfaultfd_writeprotect(ctx, arg);
|
| 4718 | break;
|
| 4719 | case UFFDIO_CONTINUE:
|
| 4720 | ret = userfaultfd_continue(ctx, arg);
|
| 4721 | break;
|
| 4722 | case UFFDIO_POISON:
|
| 4723 | ret = userfaultfd_poison(ctx, arg);
|
| 4724 | break;
|
| 4725 | case UFFDIO_RWPROTECT:
|
| 4726 | ret = userfaultfd_rwprotect(ctx, arg);
|
| 4727 | break;
|
| 4728 | case UFFDIO_SET_MODE:
|
| 4729 | ret = userfaultfd_set_mode(ctx, arg);
|
| 4730 | break;
|
| 4731 | }
|
| 4732 | return ret;
|
| 4733 | }
|
| 4734 |
|
| 4735 | #ifdef CONFIG_PROC_FS
|
| 4736 | static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
|
| 4737 | {
|
| 4738 | struct userfaultfd_ctx *ctx = f->private_data;
|
| 4739 | wait_queue_entry_t *wq;
|
| 4740 | unsigned long pending = 0, total = 0;
|
| 4741 |
|
| 4742 | spin_lock_irq(&ctx->fault_pending_wqh.lock);
|
| 4743 | list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
|
| 4744 | pending++;
|
| 4745 | total++;
|
| 4746 | }
|
| 4747 | list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
|
| 4748 | total++;
|
| 4749 | }
|
| 4750 | spin_unlock_irq(&ctx->fault_pending_wqh.lock);
|
| 4751 |
|
| 4752 | |
| 4753 | * If more protocols will be added, there will be all shown
|
| 4754 | * separated by a space. Like this:
|
| 4755 | * protocols: aa:... bb:...
|
| 4756 |
|
| 4757 | seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
|
| 4758 | pending, total, UFFD_API, userfaultfd_features(ctx),
|
| 4759 | UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
|
| 4760 | }
|
| 4761 | #endif
|
| 4762 |
|
| 4763 | static const struct file_operations userfaultfd_fops = {
|
| 4764 | #ifdef CONFIG_PROC_FS
|
| 4765 | .show_fdinfo = userfaultfd_show_fdinfo,
|
| 4766 | #endif
|
| 4767 | .release = userfaultfd_release,
|
| 4768 | .poll = userfaultfd_poll,
|
| 4769 | .read_iter = userfaultfd_read_iter,
|
| 4770 | .unlocked_ioctl = userfaultfd_ioctl,
|
| 4771 | .compat_ioctl = compat_ptr_ioctl,
|
| 4772 | .llseek = noop_llseek,
|
| 4773 | };
|
| 4774 |
|
| 4775 | static void init_once_userfaultfd_ctx(void *mem)
|
| 4776 | {
|
| 4777 | struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
|
| 4778 |
|
| 4779 | init_waitqueue_head(&ctx->fault_pending_wqh);
|
| 4780 | init_waitqueue_head(&ctx->fault_wqh);
|
| 4781 | init_waitqueue_head(&ctx->event_wqh);
|
| 4782 | init_waitqueue_head(&ctx->fd_wqh);
|
| 4783 | seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
|
| 4784 | }
|
| 4785 |
|
| 4786 | static int new_userfaultfd(int flags)
|
| 4787 | {
|
| 4788 | struct userfaultfd_ctx *ctx __free(kfree) = NULL;
|
| 4789 |
|
| 4790 | VM_WARN_ON_ONCE(!current->mm);
|
| 4791 |
|
| 4792 |
|
| 4793 | BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
|
| 4794 |
|
| 4795 | if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
|
| 4796 | return -EINVAL;
|
| 4797 |
|
| 4798 | ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
|
| 4799 | if (!ctx)
|
| 4800 | return -ENOMEM;
|
| 4801 |
|
| 4802 | refcount_set(&ctx->refcount, 1);
|
| 4803 | ctx->flags = flags;
|
| 4804 | ctx->features = 0;
|
| 4805 | ctx->released = false;
|
| 4806 | init_rwsem(&ctx->map_changing_lock);
|
| 4807 | atomic_set(&ctx->mmap_changing, 0);
|
| 4808 | ctx->mm = current->mm;
|
| 4809 |
|
| 4810 | FD_PREPARE(fdf, flags & UFFD_SHARED_FCNTL_FLAGS,
|
| 4811 | anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx,
|
| 4812 | O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS),
|
| 4813 | NULL));
|
| 4814 | if (fdf->fd < 0)
|
| 4815 | return fdf->fd;
|
| 4816 |
|
| 4817 |
|
| 4818 | mmgrab(ctx->mm);
|
| 4819 | fdf->file->f_mode |= FMODE_NOWAIT;
|
| 4820 | retain_and_null_ptr(ctx);
|
| 4821 | return fd_publish(fdf);
|
| 4822 | }
|
| 4823 |
|
| 4824 | static inline bool userfaultfd_syscall_allowed(int flags)
|
| 4825 | {
|
| 4826 |
|
| 4827 | if (flags & UFFD_USER_MODE_ONLY)
|
| 4828 | return true;
|
| 4829 |
|
| 4830 | |
| 4831 | * The user is requesting a userfaultfd which can handle kernel faults.
|
| 4832 | * Privileged users are always allowed to do this.
|
| 4833 |
|
| 4834 | if (capable(CAP_SYS_PTRACE))
|
| 4835 | return true;
|
| 4836 |
|
| 4837 |
|
| 4838 | return sysctl_unprivileged_userfaultfd;
|
| 4839 | }
|
| 4840 |
|
| 4841 | SYSCALL_DEFINE1(userfaultfd, int, flags)
|
| 4842 | {
|
| 4843 | if (!userfaultfd_syscall_allowed(flags))
|
| 4844 | return -EPERM;
|
| 4845 |
|
| 4846 | return new_userfaultfd(flags);
|
| 4847 | }
|
| 4848 |
|
| 4849 | static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)
|
| 4850 | {
|
| 4851 | if (cmd != USERFAULTFD_IOC_NEW)
|
| 4852 | return -EINVAL;
|
| 4853 |
|
| 4854 | return new_userfaultfd(flags);
|
| 4855 | }
|
| 4856 |
|
| 4857 | static const struct file_operations userfaultfd_dev_fops = {
|
| 4858 | .unlocked_ioctl = userfaultfd_dev_ioctl,
|
| 4859 | .compat_ioctl = userfaultfd_dev_ioctl,
|
| 4860 | .owner = THIS_MODULE,
|
| 4861 | .llseek = noop_llseek,
|
| 4862 | };
|
| 4863 |
|
| 4864 | static struct miscdevice userfaultfd_misc = {
|
| 4865 | .minor = MISC_DYNAMIC_MINOR,
|
| 4866 | .name = "userfaultfd",
|
| 4867 | .fops = &userfaultfd_dev_fops
|
| 4868 | };
|
| 4869 |
|
| 4870 | static int __init userfaultfd_init(void)
|
| 4871 | {
|
| 4872 | int ret;
|
| 4873 |
|
| 4874 | ret = misc_register(&userfaultfd_misc);
|
| 4875 | if (ret)
|
| 4876 | return ret;
|
| 4877 |
|
| 4878 | userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
|
| 4879 | sizeof(struct userfaultfd_ctx),
|
| 4880 | 0,
|
| 4881 | SLAB_HWCACHE_ALIGN|SLAB_PANIC,
|
| 4882 | init_once_userfaultfd_ctx);
|
| 4883 | #ifdef CONFIG_SYSCTL
|
| 4884 | register_sysctl_init("vm", vm_userfaultfd_table);
|
| 4885 | #endif
|
| 4886 | return 0;
|
| 4887 | }
|
| 4888 | __initcall(userfaultfd_init);
|