| 1 | |
| 2 | |
| 3 | * mm/rmap.c - physical to virtual reverse mappings
|
| 4 | *
|
| 5 | * Copyright 2001, Rik van Riel <[email protected]>
|
| 6 | *
|
| 7 | * Simple, low overhead reverse mapping scheme.
|
| 8 | * Please try to keep this thing as modular as possible.
|
| 9 | *
|
| 10 | * Provides methods for unmapping each kind of mapped page:
|
| 11 | * the anon methods track anonymous pages, and
|
| 12 | * the file methods track pages belonging to an inode.
|
| 13 | *
|
| 14 | * Original design by Rik van Riel <[email protected]> 2001
|
| 15 | * File methods by Dave McCracken <[email protected]> 2003, 2004
|
| 16 | * Anonymous methods by Andrea Arcangeli <[email protected]> 2004
|
| 17 | * Contributions by Hugh Dickins 2003, 2004
|
| 18 |
|
| 19 |
|
| 20 | |
| 21 | * Lock ordering in mm:
|
| 22 | *
|
| 23 | * inode->i_rwsem (while writing or truncating, not reading or faulting)
|
| 24 | * mm->mmap_lock
|
| 25 | * mapping->invalidate_lock (in filemap_fault)
|
| 26 | * folio_lock
|
| 27 | * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below)
|
| 28 | * vma_start_write
|
| 29 | * mapping->i_mmap_rwsem
|
| 30 | * anon_vma->rwsem
|
| 31 | * mm->page_table_lock or pte_lock
|
| 32 | * swap_lock (in swap_duplicate, swap_info_get)
|
| 33 | * mmlist_lock (in mmput, drain_mmlist and others)
|
| 34 | * mapping->private_lock (in block_dirty_folio)
|
| 35 | * i_pages lock (widely used)
|
| 36 | * lruvec->lru_lock (in folio_lruvec_lock_irq)
|
| 37 | * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
|
| 38 | * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
|
| 39 | * sb_lock (within inode_lock in fs/fs-writeback.c)
|
| 40 | * i_pages lock (widely used, in set_page_dirty,
|
| 41 | * in arch-dependent flush_dcache_mmap_lock,
|
| 42 | * within bdi.wb->list_lock in __sync_single_inode)
|
| 43 | *
|
| 44 | * anon_vma->rwsem,mapping->i_mmap_rwsem (memory_failure, collect_procs_anon)
|
| 45 | * ->tasklist_lock
|
| 46 | * pte map lock
|
| 47 | *
|
| 48 | * hugetlbfs PageHuge() take locks in this order:
|
| 49 | * hugetlb_fault_mutex (hugetlbfs specific page fault mutex)
|
| 50 | * vma_lock (hugetlb specific lock for pmd_sharing)
|
| 51 | * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing)
|
| 52 | * folio_lock
|
| 53 |
|
| 54 |
|
| 55 | #include <linux/mm.h>
|
| 56 | #include <linux/sched/mm.h>
|
| 57 | #include <linux/sched/task.h>
|
| 58 | #include <linux/pagemap.h>
|
| 59 | #include <linux/swap.h>
|
| 60 | #include <linux/leafops.h>
|
| 61 | #include <linux/slab.h>
|
| 62 | #include <linux/init.h>
|
| 63 | #include <linux/ksm.h>
|
| 64 | #include <linux/rmap.h>
|
| 65 | #include <linux/rcupdate.h>
|
| 66 | #include <linux/export.h>
|
| 67 | #include <linux/memcontrol.h>
|
| 68 | #include <linux/mmu_notifier.h>
|
| 69 | #include <linux/migrate.h>
|
| 70 | #include <linux/hugetlb.h>
|
| 71 | #include <linux/huge_mm.h>
|
| 72 | #include <linux/backing-dev.h>
|
| 73 | #include <linux/page_idle.h>
|
| 74 | #include <linux/memremap.h>
|
| 75 | #include <linux/userfaultfd_k.h>
|
| 76 | #include <linux/mm_inline.h>
|
| 77 | #include <linux/oom.h>
|
| 78 |
|
| 79 | #include <asm/tlb.h>
|
| 80 |
|
| 81 | #define CREATE_TRACE_POINTS
|
| 82 | #include <trace/events/migrate.h>
|
| 83 |
|
| 84 | #include "internal.h"
|
| 85 | #include "swap.h"
|
| 86 |
|
| 87 | static struct kmem_cache *anon_vma_cachep;
|
| 88 | static struct kmem_cache *anon_vma_chain_cachep;
|
| 89 |
|
| 90 | static inline struct anon_vma *anon_vma_alloc(void)
|
| 91 | {
|
| 92 | struct anon_vma *anon_vma;
|
| 93 |
|
| 94 | anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
|
| 95 | if (anon_vma) {
|
| 96 | atomic_set(&anon_vma->refcount, 1);
|
| 97 | anon_vma->num_children = 0;
|
| 98 | anon_vma->num_active_vmas = 0;
|
| 99 | anon_vma->parent = anon_vma;
|
| 100 | |
| 101 | * Initialise the anon_vma root to point to itself. If called
|
| 102 | * from fork, the root will be reset to the parents anon_vma.
|
| 103 |
|
| 104 | anon_vma->root = anon_vma;
|
| 105 | }
|
| 106 |
|
| 107 | return anon_vma;
|
| 108 | }
|
| 109 |
|
| 110 | static inline void anon_vma_free(struct anon_vma *anon_vma)
|
| 111 | {
|
| 112 | VM_BUG_ON(atomic_read(&anon_vma->refcount));
|
| 113 |
|
| 114 | |
| 115 | * Synchronize against folio_lock_anon_vma_read() such that
|
| 116 | * we can safely hold the lock without the anon_vma getting
|
| 117 | * freed.
|
| 118 | *
|
| 119 | * Relies on the full mb implied by the atomic_dec_and_test() from
|
| 120 | * put_anon_vma() against the acquire barrier implied by
|
| 121 | * down_read_trylock() from folio_lock_anon_vma_read(). This orders:
|
| 122 | *
|
| 123 | * folio_lock_anon_vma_read() VS put_anon_vma()
|
| 124 | * down_read_trylock() atomic_dec_and_test()
|
| 125 | * LOCK MB
|
| 126 | * atomic_read() rwsem_is_locked()
|
| 127 | *
|
| 128 | * LOCK should suffice since the actual taking of the lock must
|
| 129 | * happen _before_ what follows.
|
| 130 |
|
| 131 | might_sleep();
|
| 132 | if (rwsem_is_locked(&anon_vma->root->rwsem)) {
|
| 133 | anon_vma_lock_write(anon_vma);
|
| 134 | anon_vma_unlock_write(anon_vma);
|
| 135 | }
|
| 136 |
|
| 137 | kmem_cache_free(anon_vma_cachep, anon_vma);
|
| 138 | }
|
| 139 |
|
| 140 | static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
|
| 141 | {
|
| 142 | return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
|
| 143 | }
|
| 144 |
|
| 145 | static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
|
| 146 | {
|
| 147 | kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
|
| 148 | }
|
| 149 |
|
| 150 | static void anon_vma_chain_assign(struct vm_area_struct *vma,
|
| 151 | struct anon_vma_chain *avc,
|
| 152 | struct anon_vma *anon_vma)
|
| 153 | {
|
| 154 | avc->vma = vma;
|
| 155 | avc->anon_vma = anon_vma;
|
| 156 | list_add(&avc->same_vma, &vma->anon_vma_chain);
|
| 157 | }
|
| 158 |
|
| 159 | |
| 160 | * __anon_vma_prepare - attach an anon_vma to a memory region
|
| 161 | * @vma: the memory region in question
|
| 162 | *
|
| 163 | * This makes sure the memory mapping described by 'vma' has
|
| 164 | * an 'anon_vma' attached to it, so that we can associate the
|
| 165 | * anonymous pages mapped into it with that anon_vma.
|
| 166 | *
|
| 167 | * The common case will be that we already have one, which
|
| 168 | * is handled inline by anon_vma_prepare(). But if
|
| 169 | * not we either need to find an adjacent mapping that we
|
| 170 | * can re-use the anon_vma from (very common when the only
|
| 171 | * reason for splitting a vma has been mprotect()), or we
|
| 172 | * allocate a new one.
|
| 173 | *
|
| 174 | * Anon-vma allocations are very subtle, because we may have
|
| 175 | * optimistically looked up an anon_vma in folio_lock_anon_vma_read()
|
| 176 | * and that may actually touch the rwsem even in the newly
|
| 177 | * allocated vma (it depends on RCU to make sure that the
|
| 178 | * anon_vma isn't actually destroyed).
|
| 179 | *
|
| 180 | * As a result, we need to do proper anon_vma locking even
|
| 181 | * for the new allocation. At the same time, we do not want
|
| 182 | * to do any locking for the common case of already having
|
| 183 | * an anon_vma.
|
| 184 |
|
| 185 | int __anon_vma_prepare(struct vm_area_struct *vma)
|
| 186 | {
|
| 187 | struct mm_struct *mm = vma->vm_mm;
|
| 188 | struct anon_vma *anon_vma, *allocated;
|
| 189 | struct anon_vma_chain *avc;
|
| 190 |
|
| 191 | mmap_assert_locked(mm);
|
| 192 | might_sleep();
|
| 193 |
|
| 194 | avc = anon_vma_chain_alloc(GFP_KERNEL);
|
| 195 | if (!avc)
|
| 196 | goto out_enomem;
|
| 197 |
|
| 198 | anon_vma = find_mergeable_anon_vma(vma);
|
| 199 | allocated = NULL;
|
| 200 | if (!anon_vma) {
|
| 201 | anon_vma = anon_vma_alloc();
|
| 202 | if (unlikely(!anon_vma))
|
| 203 | goto out_enomem_free_avc;
|
| 204 | anon_vma->num_children++;
|
| 205 | allocated = anon_vma;
|
| 206 | }
|
| 207 |
|
| 208 | anon_vma_lock_write(anon_vma);
|
| 209 |
|
| 210 | spin_lock(&mm->page_table_lock);
|
| 211 | if (likely(!vma->anon_vma)) {
|
| 212 | |
| 213 | * Make anon_vma fields visible before anon_vma is published.
|
| 214 | * Paired with an address dependency in reusable_anon_vma().
|
| 215 |
|
| 216 | smp_store_release(&vma->anon_vma, anon_vma);
|
| 217 | anon_vma_chain_assign(vma, avc, anon_vma);
|
| 218 | anon_rmap_tree_insert(avc, anon_vma);
|
| 219 | anon_vma->num_active_vmas++;
|
| 220 | allocated = NULL;
|
| 221 | avc = NULL;
|
| 222 | }
|
| 223 | spin_unlock(&mm->page_table_lock);
|
| 224 | anon_vma_unlock_write(anon_vma);
|
| 225 |
|
| 226 | if (unlikely(allocated))
|
| 227 | put_anon_vma(allocated);
|
| 228 | if (unlikely(avc))
|
| 229 | anon_vma_chain_free(avc);
|
| 230 |
|
| 231 | return 0;
|
| 232 |
|
| 233 | out_enomem_free_avc:
|
| 234 | anon_vma_chain_free(avc);
|
| 235 | out_enomem:
|
| 236 | return -ENOMEM;
|
| 237 | }
|
| 238 |
|
| 239 | static void check_anon_vma_clone(struct vm_area_struct *dst,
|
| 240 | struct vm_area_struct *src,
|
| 241 | enum vma_operation operation)
|
| 242 | {
|
| 243 |
|
| 244 | mmap_assert_write_locked(src->vm_mm);
|
| 245 |
|
| 246 | VM_WARN_ON_ONCE(operation != VMA_OP_FORK && dst->vm_mm != src->vm_mm);
|
| 247 |
|
| 248 |
|
| 249 | VM_WARN_ON_ONCE(!src->anon_vma && !list_empty(&src->anon_vma_chain));
|
| 250 | VM_WARN_ON_ONCE(!src->anon_vma && dst->anon_vma);
|
| 251 |
|
| 252 | VM_WARN_ON_ONCE(!list_empty(&dst->anon_vma_chain));
|
| 253 | |
| 254 | * On fork, dst->anon_vma is set NULL (temporarily). Otherwise, anon_vma
|
| 255 | * must be the same across dst and src.
|
| 256 |
|
| 257 | VM_WARN_ON_ONCE(dst->anon_vma && dst->anon_vma != src->anon_vma);
|
| 258 | |
| 259 | * Essentially equivalent to above - if not a no-op, we should expect
|
| 260 | * dst->anon_vma to be set for everything except a fork.
|
| 261 |
|
| 262 | VM_WARN_ON_ONCE(operation != VMA_OP_FORK && src->anon_vma &&
|
| 263 | !dst->anon_vma);
|
| 264 |
|
| 265 | VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED &&
|
| 266 | !list_is_singular(&src->anon_vma_chain));
|
| 267 | #ifdef CONFIG_PER_VMA_LOCK
|
| 268 |
|
| 269 | VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED &&
|
| 270 | vma_is_attached(dst));
|
| 271 | #endif
|
| 272 | }
|
| 273 |
|
| 274 | static void maybe_reuse_anon_vma(struct vm_area_struct *dst,
|
| 275 | struct anon_vma *anon_vma)
|
| 276 | {
|
| 277 |
|
| 278 | if (dst->anon_vma)
|
| 279 | return;
|
| 280 |
|
| 281 | |
| 282 | * We reuse an anon_vma if any linking VMAs were unmapped and it has
|
| 283 | * only a single child at most.
|
| 284 |
|
| 285 | if (anon_vma->num_active_vmas > 0)
|
| 286 | return;
|
| 287 | if (anon_vma->num_children > 1)
|
| 288 | return;
|
| 289 |
|
| 290 | dst->anon_vma = anon_vma;
|
| 291 | anon_vma->num_active_vmas++;
|
| 292 | }
|
| 293 |
|
| 294 | static void cleanup_partial_anon_vmas(struct vm_area_struct *vma);
|
| 295 |
|
| 296 | |
| 297 | * anon_vma_clone - Establishes new anon_vma_chain objects in @dst linking to
|
| 298 | * all of the anon_vma objects contained within @src anon_vma_chain's.
|
| 299 | * @dst: The destination VMA with an empty anon_vma_chain.
|
| 300 | * @src: The source VMA we wish to duplicate.
|
| 301 | * @operation: The type of operation which resulted in the clone.
|
| 302 | *
|
| 303 | * This is the heart of the VMA side of the anon_vma implementation - we invoke
|
| 304 | * this function whenever we need to set up a new VMA's anon_vma state.
|
| 305 | *
|
| 306 | * This is invoked for:
|
| 307 | *
|
| 308 | * - VMA Merge, but only when @dst is unfaulted and @src is faulted - meaning we
|
| 309 | * clone @src into @dst.
|
| 310 | * - VMA split.
|
| 311 | * - VMA (m)remap.
|
| 312 | * - Fork of faulted VMA.
|
| 313 | *
|
| 314 | * In all cases other than fork this is simply a duplication. Fork additionally
|
| 315 | * adds a new active anon_vma.
|
| 316 | *
|
| 317 | * ONLY in the case of fork do we try to 'reuse' existing anon_vma's in an
|
| 318 | * anon_vma hierarchy, reusing anon_vma's which have no VMA associated with them
|
| 319 | * but do have a single child. This is to avoid waste of memory when repeatedly
|
| 320 | * forking.
|
| 321 | *
|
| 322 | * Returns: 0 on success, -ENOMEM on failure.
|
| 323 |
|
| 324 | int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
|
| 325 | enum vma_operation operation)
|
| 326 | {
|
| 327 | struct anon_vma_chain *avc, *pavc;
|
| 328 | struct anon_vma *active_anon_vma = src->anon_vma;
|
| 329 |
|
| 330 | check_anon_vma_clone(dst, src, operation);
|
| 331 |
|
| 332 | if (!active_anon_vma)
|
| 333 | return 0;
|
| 334 |
|
| 335 | |
| 336 | * Allocate AVCs. We don't need an anon_vma lock for this as we
|
| 337 | * are not updating the anon_vma rbtree nor are we changing
|
| 338 | * anon_vma statistics.
|
| 339 | *
|
| 340 | * Either src, dst have the same mm for which we hold an exclusive mmap
|
| 341 | * write lock, or we are forking and we hold it on src->vm_mm and dst is
|
| 342 | * not yet accessible to other threads so there's no possibliity of the
|
| 343 | * unlinked AVC's being observed yet.
|
| 344 |
|
| 345 | list_for_each_entry(pavc, &src->anon_vma_chain, same_vma) {
|
| 346 | avc = anon_vma_chain_alloc(GFP_KERNEL);
|
| 347 | if (!avc)
|
| 348 | goto enomem_failure;
|
| 349 |
|
| 350 | anon_vma_chain_assign(dst, avc, pavc->anon_vma);
|
| 351 | }
|
| 352 |
|
| 353 | |
| 354 | * Now link the anon_vma's back to the newly inserted AVCs.
|
| 355 | * Note that all anon_vma's share the same root.
|
| 356 |
|
| 357 | anon_vma_lock_write(active_anon_vma);
|
| 358 | list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) {
|
| 359 | struct anon_vma *anon_vma = avc->anon_vma;
|
| 360 |
|
| 361 | anon_rmap_tree_insert(avc, anon_vma);
|
| 362 | if (operation == VMA_OP_FORK)
|
| 363 | maybe_reuse_anon_vma(dst, anon_vma);
|
| 364 | }
|
| 365 |
|
| 366 | if (operation != VMA_OP_FORK)
|
| 367 | dst->anon_vma->num_active_vmas++;
|
| 368 |
|
| 369 | anon_vma_unlock_write(active_anon_vma);
|
| 370 | return 0;
|
| 371 |
|
| 372 | enomem_failure:
|
| 373 | cleanup_partial_anon_vmas(dst);
|
| 374 | return -ENOMEM;
|
| 375 | }
|
| 376 |
|
| 377 | |
| 378 | * Attach vma to its own anon_vma, as well as to the anon_vmas that
|
| 379 | * the corresponding VMA in the parent process is attached to.
|
| 380 | * Returns 0 on success, non-zero on failure.
|
| 381 |
|
| 382 | int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
|
| 383 | {
|
| 384 | struct anon_vma_chain *avc;
|
| 385 | struct anon_vma *anon_vma;
|
| 386 | int rc;
|
| 387 |
|
| 388 |
|
| 389 | if (!pvma->anon_vma)
|
| 390 | return 0;
|
| 391 |
|
| 392 |
|
| 393 | vma->anon_vma = NULL;
|
| 394 |
|
| 395 | anon_vma = anon_vma_alloc();
|
| 396 | if (!anon_vma)
|
| 397 | return -ENOMEM;
|
| 398 | avc = anon_vma_chain_alloc(GFP_KERNEL);
|
| 399 | if (!avc) {
|
| 400 | put_anon_vma(anon_vma);
|
| 401 | return -ENOMEM;
|
| 402 | }
|
| 403 |
|
| 404 | |
| 405 | * First, attach the new VMA to the parent VMA's anon_vmas,
|
| 406 | * so rmap can find non-COWed pages in child processes.
|
| 407 |
|
| 408 | rc = anon_vma_clone(vma, pvma, VMA_OP_FORK);
|
| 409 |
|
| 410 | if (rc || vma->anon_vma) {
|
| 411 | put_anon_vma(anon_vma);
|
| 412 | anon_vma_chain_free(avc);
|
| 413 | return rc;
|
| 414 | }
|
| 415 |
|
| 416 | |
| 417 | * OK no reuse, so add our own anon_vma.
|
| 418 | *
|
| 419 | * Since it is not linked anywhere we can safely manipulate anon_vma
|
| 420 | * fields without a lock.
|
| 421 |
|
| 422 |
|
| 423 | anon_vma->num_active_vmas = 1;
|
| 424 | |
| 425 | * The root anon_vma's rwsem is the lock actually used when we
|
| 426 | * lock any of the anon_vmas in this anon_vma tree.
|
| 427 |
|
| 428 | anon_vma->root = pvma->anon_vma->root;
|
| 429 | anon_vma->parent = pvma->anon_vma;
|
| 430 | |
| 431 | * With refcounts, an anon_vma can stay around longer than the
|
| 432 | * process it belongs to. The root anon_vma needs to be pinned until
|
| 433 | * this anon_vma is freed, because the lock lives in the root.
|
| 434 |
|
| 435 | get_anon_vma(anon_vma->root);
|
| 436 |
|
| 437 | vma->anon_vma = anon_vma;
|
| 438 | anon_vma_chain_assign(vma, avc, anon_vma);
|
| 439 |
|
| 440 | anon_vma_lock_write(anon_vma);
|
| 441 | anon_rmap_tree_insert(avc, anon_vma);
|
| 442 | anon_vma->parent->num_children++;
|
| 443 | anon_vma_unlock_write(anon_vma);
|
| 444 |
|
| 445 | return 0;
|
| 446 | }
|
| 447 |
|
| 448 | |
| 449 | * In the unfortunate case of anon_vma_clone() failing to allocate memory we
|
| 450 | * have to clean things up.
|
| 451 | *
|
| 452 | * Since we allocate anon_vma_chain's before we insert them into the interval
|
| 453 | * trees, we simply have to free up the AVC's and remove the entries from the
|
| 454 | * VMA's anon_vma_chain.
|
| 455 |
|
| 456 | static void cleanup_partial_anon_vmas(struct vm_area_struct *vma)
|
| 457 | {
|
| 458 | struct anon_vma_chain *avc, *next;
|
| 459 |
|
| 460 | list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
|
| 461 | list_del(&avc->same_vma);
|
| 462 | anon_vma_chain_free(avc);
|
| 463 | }
|
| 464 |
|
| 465 | |
| 466 | * The anon_vma assigned to this VMA is no longer valid, as we were not
|
| 467 | * able to correctly clone AVC state. Avoid inconsistent anon_vma tree
|
| 468 | * state by resetting.
|
| 469 |
|
| 470 | vma->anon_vma = NULL;
|
| 471 | }
|
| 472 |
|
| 473 | |
| 474 | * unlink_anon_vmas() - remove all links between a VMA and anon_vma's, freeing
|
| 475 | * anon_vma_chain objects.
|
| 476 | * @vma: The VMA whose links to anon_vma objects is to be severed.
|
| 477 | *
|
| 478 | * As part of the process anon_vma_chain's are freed,
|
| 479 | * anon_vma->num_children,num_active_vmas is updated as required and, if the
|
| 480 | * relevant anon_vma references no further VMAs, its reference count is
|
| 481 | * decremented.
|
| 482 |
|
| 483 | void unlink_anon_vmas(struct vm_area_struct *vma)
|
| 484 | {
|
| 485 | struct anon_vma_chain *avc, *next;
|
| 486 | struct anon_vma *active_anon_vma = vma->anon_vma;
|
| 487 |
|
| 488 |
|
| 489 | mmap_assert_locked(vma->vm_mm);
|
| 490 |
|
| 491 |
|
| 492 | if (!active_anon_vma) {
|
| 493 | VM_WARN_ON_ONCE(!list_empty(&vma->anon_vma_chain));
|
| 494 | return;
|
| 495 | }
|
| 496 |
|
| 497 | anon_vma_lock_write(active_anon_vma);
|
| 498 |
|
| 499 | |
| 500 | * Unlink each anon_vma chained to the VMA. This list is ordered
|
| 501 | * from newest to oldest, ensuring the root anon_vma gets freed last.
|
| 502 |
|
| 503 | list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
|
| 504 | struct anon_vma *anon_vma = avc->anon_vma;
|
| 505 |
|
| 506 | anon_rmap_tree_remove(avc, anon_vma);
|
| 507 |
|
| 508 | |
| 509 | * Leave empty anon_vmas on the list - we'll need
|
| 510 | * to free them outside the lock.
|
| 511 |
|
| 512 | if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
|
| 513 | anon_vma->parent->num_children--;
|
| 514 | continue;
|
| 515 | }
|
| 516 |
|
| 517 | list_del(&avc->same_vma);
|
| 518 | anon_vma_chain_free(avc);
|
| 519 | }
|
| 520 |
|
| 521 | active_anon_vma->num_active_vmas--;
|
| 522 | |
| 523 | * vma would still be needed after unlink, and anon_vma will be prepared
|
| 524 | * when handle fault.
|
| 525 |
|
| 526 | vma->anon_vma = NULL;
|
| 527 | anon_vma_unlock_write(active_anon_vma);
|
| 528 |
|
| 529 |
|
| 530 | |
| 531 | * Iterate the list once more, it now only contains empty and unlinked
|
| 532 | * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
|
| 533 | * needing to write-acquire the anon_vma->root->rwsem.
|
| 534 |
|
| 535 | list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
|
| 536 | struct anon_vma *anon_vma = avc->anon_vma;
|
| 537 |
|
| 538 | VM_WARN_ON(anon_vma->num_children);
|
| 539 | VM_WARN_ON(anon_vma->num_active_vmas);
|
| 540 | put_anon_vma(anon_vma);
|
| 541 |
|
| 542 | list_del(&avc->same_vma);
|
| 543 | anon_vma_chain_free(avc);
|
| 544 | }
|
| 545 | }
|
| 546 |
|
| 547 | static void anon_vma_ctor(void *data)
|
| 548 | {
|
| 549 | struct anon_vma *anon_vma = data;
|
| 550 |
|
| 551 | init_rwsem(&anon_vma->rwsem);
|
| 552 | atomic_set(&anon_vma->refcount, 0);
|
| 553 | anon_vma->rb_root = RB_ROOT_CACHED;
|
| 554 | }
|
| 555 |
|
| 556 | void __init anon_vma_init(void)
|
| 557 | {
|
| 558 | anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
|
| 559 | 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
|
| 560 | anon_vma_ctor);
|
| 561 | anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
|
| 562 | SLAB_PANIC|SLAB_ACCOUNT);
|
| 563 | }
|
| 564 |
|
| 565 | |
| 566 | * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
|
| 567 | *
|
| 568 | * Since there is no serialization what so ever against folio_remove_rmap_*()
|
| 569 | * the best this function can do is return a refcount increased anon_vma
|
| 570 | * that might have been relevant to this page.
|
| 571 | *
|
| 572 | * The page might have been remapped to a different anon_vma or the anon_vma
|
| 573 | * returned may already be freed (and even reused).
|
| 574 | *
|
| 575 | * In case it was remapped to a different anon_vma, the new anon_vma will be a
|
| 576 | * child of the old anon_vma, and the anon_vma lifetime rules will therefore
|
| 577 | * ensure that any anon_vma obtained from the page will still be valid for as
|
| 578 | * long as we observe folio_mapped() [ hence all those folio_mapped() tests ].
|
| 579 | *
|
| 580 | * All users of this function must be very careful when walking the anon_vma
|
| 581 | * chain and verify that the page in question is indeed mapped in it
|
| 582 | * [ something equivalent to page_mapped_in_vma() ].
|
| 583 | *
|
| 584 | * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
|
| 585 | * folio_remove_rmap_*() that the anon_vma pointer from page->mapping is valid
|
| 586 | * if there is a mapcount, we can dereference the anon_vma after observing
|
| 587 | * those.
|
| 588 | *
|
| 589 | * NOTE: the caller should hold folio lock when calling this.
|
| 590 |
|
| 591 | struct anon_vma *folio_get_anon_vma(const struct folio *folio)
|
| 592 | {
|
| 593 | struct anon_vma *anon_vma = NULL;
|
| 594 | unsigned long anon_mapping;
|
| 595 |
|
| 596 | VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 597 |
|
| 598 | rcu_read_lock();
|
| 599 | anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
|
| 600 | if ((anon_mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON)
|
| 601 | goto out;
|
| 602 | if (!folio_mapped(folio))
|
| 603 | goto out;
|
| 604 |
|
| 605 | anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON);
|
| 606 | if (!atomic_inc_not_zero(&anon_vma->refcount)) {
|
| 607 | anon_vma = NULL;
|
| 608 | goto out;
|
| 609 | }
|
| 610 |
|
| 611 | |
| 612 | * If this folio is still mapped, then its anon_vma cannot have been
|
| 613 | * freed. But if it has been unmapped, we have no security against the
|
| 614 | * anon_vma structure being freed and reused (for another anon_vma:
|
| 615 | * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
|
| 616 | * above cannot corrupt).
|
| 617 |
|
| 618 | if (!folio_mapped(folio)) {
|
| 619 | rcu_read_unlock();
|
| 620 | put_anon_vma(anon_vma);
|
| 621 | return NULL;
|
| 622 | }
|
| 623 | out:
|
| 624 | rcu_read_unlock();
|
| 625 |
|
| 626 | return anon_vma;
|
| 627 | }
|
| 628 |
|
| 629 | |
| 630 | * Similar to folio_get_anon_vma() except it locks the anon_vma.
|
| 631 | *
|
| 632 | * Its a little more complex as it tries to keep the fast path to a single
|
| 633 | * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
|
| 634 | * reference like with folio_get_anon_vma() and then block on the mutex
|
| 635 | * on !rwc->try_lock case.
|
| 636 |
|
| 637 | struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
|
| 638 | struct rmap_walk_control *rwc)
|
| 639 | {
|
| 640 | struct anon_vma *anon_vma = NULL;
|
| 641 | struct anon_vma *root_anon_vma;
|
| 642 | unsigned long anon_mapping;
|
| 643 |
|
| 644 | VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 645 |
|
| 646 | rcu_read_lock();
|
| 647 | anon_mapping = (unsigned long)READ_ONCE(folio->mapping);
|
| 648 | if ((anon_mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON)
|
| 649 | goto out;
|
| 650 | if (!folio_mapped(folio))
|
| 651 | goto out;
|
| 652 |
|
| 653 | anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON);
|
| 654 | root_anon_vma = READ_ONCE(anon_vma->root);
|
| 655 | if (down_read_trylock(&root_anon_vma->rwsem)) {
|
| 656 | |
| 657 | * If the folio is still mapped, then this anon_vma is still
|
| 658 | * its anon_vma, and holding the mutex ensures that it will
|
| 659 | * not go away, see anon_vma_free().
|
| 660 |
|
| 661 | if (!folio_mapped(folio)) {
|
| 662 | up_read(&root_anon_vma->rwsem);
|
| 663 | anon_vma = NULL;
|
| 664 | }
|
| 665 | goto out;
|
| 666 | }
|
| 667 |
|
| 668 | if (rwc && rwc->try_lock) {
|
| 669 | anon_vma = NULL;
|
| 670 | rwc->contended = true;
|
| 671 | goto out;
|
| 672 | }
|
| 673 |
|
| 674 |
|
| 675 | if (!atomic_inc_not_zero(&anon_vma->refcount)) {
|
| 676 | anon_vma = NULL;
|
| 677 | goto out;
|
| 678 | }
|
| 679 |
|
| 680 | if (!folio_mapped(folio)) {
|
| 681 | rcu_read_unlock();
|
| 682 | put_anon_vma(anon_vma);
|
| 683 | return NULL;
|
| 684 | }
|
| 685 |
|
| 686 |
|
| 687 | rcu_read_unlock();
|
| 688 | anon_vma_lock_read(anon_vma);
|
| 689 |
|
| 690 | if (atomic_dec_and_test(&anon_vma->refcount)) {
|
| 691 | |
| 692 | * Oops, we held the last refcount, release the lock
|
| 693 | * and bail -- can't simply use put_anon_vma() because
|
| 694 | * we'll deadlock on the anon_vma_lock_write() recursion.
|
| 695 |
|
| 696 | anon_vma_unlock_read(anon_vma);
|
| 697 | __put_anon_vma(anon_vma);
|
| 698 | anon_vma = NULL;
|
| 699 | }
|
| 700 |
|
| 701 | return anon_vma;
|
| 702 |
|
| 703 | out:
|
| 704 | rcu_read_unlock();
|
| 705 | return anon_vma;
|
| 706 | }
|
| 707 |
|
| 708 | #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
|
| 709 | |
| 710 | * Flush TLB entries for recently unmapped pages from remote CPUs. It is
|
| 711 | * important if a PTE was dirty when it was unmapped that it's flushed
|
| 712 | * before any IO is initiated on the page to prevent lost writes. Similarly,
|
| 713 | * it must be flushed before freeing to prevent data leakage.
|
| 714 |
|
| 715 | void try_to_unmap_flush(void)
|
| 716 | {
|
| 717 | struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
|
| 718 |
|
| 719 | if (!tlb_ubc->flush_required)
|
| 720 | return;
|
| 721 |
|
| 722 | arch_tlbbatch_flush(&tlb_ubc->arch);
|
| 723 | tlb_ubc->flush_required = false;
|
| 724 | tlb_ubc->writable = false;
|
| 725 | }
|
| 726 |
|
| 727 |
|
| 728 | void try_to_unmap_flush_dirty(void)
|
| 729 | {
|
| 730 | struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
|
| 731 |
|
| 732 | if (tlb_ubc->writable)
|
| 733 | try_to_unmap_flush();
|
| 734 | }
|
| 735 |
|
| 736 | |
| 737 | * Bits 0-14 of mm->tlb_flush_batched record pending generations.
|
| 738 | * Bits 16-30 of mm->tlb_flush_batched bit record flushed generations.
|
| 739 |
|
| 740 | #define TLB_FLUSH_BATCH_FLUSHED_SHIFT 16
|
| 741 | #define TLB_FLUSH_BATCH_PENDING_MASK \
|
| 742 | ((1 << (TLB_FLUSH_BATCH_FLUSHED_SHIFT - 1)) - 1)
|
| 743 | #define TLB_FLUSH_BATCH_PENDING_LARGE \
|
| 744 | (TLB_FLUSH_BATCH_PENDING_MASK / 2)
|
| 745 |
|
| 746 | static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
|
| 747 | unsigned long start, unsigned long end)
|
| 748 | {
|
| 749 | struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
|
| 750 | int batch;
|
| 751 | bool writable = pte_dirty(pteval);
|
| 752 |
|
| 753 | if (!pte_accessible(mm, pteval))
|
| 754 | return;
|
| 755 |
|
| 756 | arch_tlbbatch_add_pending(&tlb_ubc->arch, mm, start, end);
|
| 757 | tlb_ubc->flush_required = true;
|
| 758 |
|
| 759 | |
| 760 | * Ensure compiler does not re-order the setting of tlb_flush_batched
|
| 761 | * before the PTE is cleared.
|
| 762 |
|
| 763 | barrier();
|
| 764 | batch = atomic_read(&mm->tlb_flush_batched);
|
| 765 | retry:
|
| 766 | if ((batch & TLB_FLUSH_BATCH_PENDING_MASK) > TLB_FLUSH_BATCH_PENDING_LARGE) {
|
| 767 | |
| 768 | * Prevent `pending' from catching up with `flushed' because of
|
| 769 | * overflow. Reset `pending' and `flushed' to be 1 and 0 if
|
| 770 | * `pending' becomes large.
|
| 771 |
|
| 772 | if (!atomic_try_cmpxchg(&mm->tlb_flush_batched, &batch, 1))
|
| 773 | goto retry;
|
| 774 | } else {
|
| 775 | atomic_inc(&mm->tlb_flush_batched);
|
| 776 | }
|
| 777 |
|
| 778 | |
| 779 | * If the PTE was dirty then it's best to assume it's writable. The
|
| 780 | * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
|
| 781 | * before the page is queued for IO.
|
| 782 |
|
| 783 | if (writable)
|
| 784 | tlb_ubc->writable = true;
|
| 785 | }
|
| 786 |
|
| 787 | |
| 788 | * Returns true if the TLB flush should be deferred to the end of a batch of
|
| 789 | * unmap operations to reduce IPIs.
|
| 790 |
|
| 791 | static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
|
| 792 | {
|
| 793 | if (!(flags & TTU_BATCH_FLUSH))
|
| 794 | return false;
|
| 795 |
|
| 796 | return arch_tlbbatch_should_defer(mm);
|
| 797 | }
|
| 798 |
|
| 799 | |
| 800 | * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
|
| 801 | * releasing the PTL if TLB flushes are batched. It's possible for a parallel
|
| 802 | * operation such as mprotect or munmap to race between reclaim unmapping
|
| 803 | * the page and flushing the page. If this race occurs, it potentially allows
|
| 804 | * access to data via a stale TLB entry. Tracking all mm's that have TLB
|
| 805 | * batching in flight would be expensive during reclaim so instead track
|
| 806 | * whether TLB batching occurred in the past and if so then do a flush here
|
| 807 | * if required. This will cost one additional flush per reclaim cycle paid
|
| 808 | * by the first operation at risk such as mprotect and mumap.
|
| 809 | *
|
| 810 | * This must be called under the PTL so that an access to tlb_flush_batched
|
| 811 | * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
|
| 812 | * via the PTL.
|
| 813 |
|
| 814 | void flush_tlb_batched_pending(struct mm_struct *mm)
|
| 815 | {
|
| 816 | int batch = atomic_read(&mm->tlb_flush_batched);
|
| 817 | int pending = batch & TLB_FLUSH_BATCH_PENDING_MASK;
|
| 818 | int flushed = batch >> TLB_FLUSH_BATCH_FLUSHED_SHIFT;
|
| 819 |
|
| 820 | if (pending != flushed) {
|
| 821 | flush_tlb_mm(mm);
|
| 822 | |
| 823 | * If the new TLB flushing is pending during flushing, leave
|
| 824 | * mm->tlb_flush_batched as is, to avoid losing flushing.
|
| 825 |
|
| 826 | atomic_cmpxchg(&mm->tlb_flush_batched, batch,
|
| 827 | pending | (pending << TLB_FLUSH_BATCH_FLUSHED_SHIFT));
|
| 828 | }
|
| 829 | }
|
| 830 | #else
|
| 831 | static void set_tlb_ubc_flush_pending(struct mm_struct *mm, pte_t pteval,
|
| 832 | unsigned long start, unsigned long end)
|
| 833 | {
|
| 834 | }
|
| 835 |
|
| 836 | static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
|
| 837 | {
|
| 838 | return false;
|
| 839 | }
|
| 840 | #endif
|
| 841 |
|
| 842 | |
| 843 | * page_address_in_vma - The virtual address of a page in this VMA.
|
| 844 | * @folio: The folio containing the page.
|
| 845 | * @page: The page within the folio.
|
| 846 | * @vma: The VMA we need to know the address in.
|
| 847 | *
|
| 848 | * Calculates the user virtual address of this page in the specified VMA.
|
| 849 | * It is the caller's responsibility to check the page is actually
|
| 850 | * within the VMA. There may not currently be a PTE pointing at this
|
| 851 | * page, but if a page fault occurs at this address, this is the page
|
| 852 | * which will be accessed.
|
| 853 | *
|
| 854 | * Context: Caller should hold a reference to the folio. Caller should
|
| 855 | * hold a lock (eg the i_mmap_lock or the mmap_lock) which keeps the
|
| 856 | * VMA from being altered.
|
| 857 | *
|
| 858 | * Return: The virtual address corresponding to this page in the VMA.
|
| 859 |
|
| 860 | unsigned long page_address_in_vma(const struct folio *folio,
|
| 861 | const struct page *page, const struct vm_area_struct *vma)
|
| 862 | {
|
| 863 | if (folio_test_anon(folio)) {
|
| 864 | struct anon_vma *anon_vma = folio_anon_vma(folio);
|
| 865 | |
| 866 | * Note: swapoff's unuse_vma() is more efficient with this
|
| 867 | * check, and needs it to match anon_vma when KSM is active.
|
| 868 |
|
| 869 | if (!vma->anon_vma || !anon_vma ||
|
| 870 | vma->anon_vma->root != anon_vma->root)
|
| 871 | return -EFAULT;
|
| 872 |
|
| 873 | return vma_anon_address(vma, page_pgoff(folio, page), 1);
|
| 874 | } else if (!vma->vm_file) {
|
| 875 | return -EFAULT;
|
| 876 | } else if (vma->vm_file->f_mapping != folio->mapping) {
|
| 877 | return -EFAULT;
|
| 878 | }
|
| 879 |
|
| 880 | return vma_filebacked_address(vma, page_pgoff(folio, page), 1);
|
| 881 | }
|
| 882 |
|
| 883 | |
| 884 | * Returns the actual pmd_t* where we expect 'address' to be mapped from, or
|
| 885 | * NULL if it doesn't exist. No guarantees / checks on what the pmd_t*
|
| 886 | * represents.
|
| 887 |
|
| 888 | pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
|
| 889 | {
|
| 890 | pgd_t *pgd;
|
| 891 | p4d_t *p4d;
|
| 892 | pud_t *pud;
|
| 893 | pmd_t *pmd = NULL;
|
| 894 |
|
| 895 | pgd = pgd_offset(mm, address);
|
| 896 | if (!pgd_present(*pgd))
|
| 897 | goto out;
|
| 898 |
|
| 899 | p4d = p4d_offset(pgd, address);
|
| 900 | if (!p4d_present(*p4d))
|
| 901 | goto out;
|
| 902 |
|
| 903 | pud = pud_offset(p4d, address);
|
| 904 | if (!pud_present(*pud))
|
| 905 | goto out;
|
| 906 |
|
| 907 | pmd = pmd_offset(pud, address);
|
| 908 | out:
|
| 909 | return pmd;
|
| 910 | }
|
| 911 |
|
| 912 | struct folio_referenced_arg {
|
| 913 | int mapcount;
|
| 914 | int referenced;
|
| 915 | vma_flags_t vma_flags;
|
| 916 | struct mem_cgroup *memcg;
|
| 917 | };
|
| 918 |
|
| 919 | |
| 920 | * arg: folio_referenced_arg will be passed
|
| 921 |
|
| 922 | static bool folio_referenced_one(struct folio *folio,
|
| 923 | struct vm_area_struct *vma, unsigned long address, void *arg)
|
| 924 | {
|
| 925 | struct folio_referenced_arg *pra = arg;
|
| 926 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
|
| 927 | int ptes = 0, referenced = 0;
|
| 928 | unsigned int nr;
|
| 929 |
|
| 930 | while (page_vma_mapped_walk(&pvmw)) {
|
| 931 | address = pvmw.address;
|
| 932 | nr = 1;
|
| 933 |
|
| 934 | if (vma_test(vma, VMA_LOCKED_BIT)) {
|
| 935 | ptes++;
|
| 936 | pra->mapcount--;
|
| 937 |
|
| 938 |
|
| 939 | if (pvmw.pte && ptes != pvmw.nr_pages)
|
| 940 | continue;
|
| 941 |
|
| 942 | |
| 943 | * All PTEs must be protected by page table lock in
|
| 944 | * order to mlock the page.
|
| 945 | *
|
| 946 | * If page table boundary has been cross, current ptl
|
| 947 | * only protect part of ptes.
|
| 948 |
|
| 949 | if (pvmw.flags & PVMW_PGTABLE_CROSSED)
|
| 950 | continue;
|
| 951 |
|
| 952 |
|
| 953 | mlock_vma_folio(folio, vma);
|
| 954 | page_vma_mapped_walk_done(&pvmw);
|
| 955 | vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT);
|
| 956 | return false;
|
| 957 | }
|
| 958 |
|
| 959 | |
| 960 | * Skip the non-shared swapbacked folio mapped solely by
|
| 961 | * the exiting or OOM-reaped process. This avoids redundant
|
| 962 | * swap-out followed by an immediate unmap.
|
| 963 |
|
| 964 | if ((!atomic_read(&vma->vm_mm->mm_users) ||
|
| 965 | check_stable_address_space(vma->vm_mm)) &&
|
| 966 | folio_test_anon(folio) && folio_test_swapbacked(folio) &&
|
| 967 | !folio_maybe_mapped_shared(folio)) {
|
| 968 | pra->referenced = -1;
|
| 969 | page_vma_mapped_walk_done(&pvmw);
|
| 970 | return false;
|
| 971 | }
|
| 972 |
|
| 973 | if (pvmw.pte && folio_test_large(folio)) {
|
| 974 | const unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
|
| 975 | const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
|
| 976 | pte_t pteval = ptep_get(pvmw.pte);
|
| 977 |
|
| 978 | nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr);
|
| 979 | }
|
| 980 |
|
| 981 | |
| 982 | * When LRU is switching, we don’t know where the surrounding folios
|
| 983 | * are. —they could be on active/inactive lists or on MGLRU. So the
|
| 984 | * simplest approach is to disable this look-around optimization.
|
| 985 |
|
| 986 | if (lru_gen_enabled() && !lru_gen_switching() && pvmw.pte) {
|
| 987 | if (lru_gen_look_around(&pvmw, nr))
|
| 988 | referenced++;
|
| 989 | } else if (pvmw.pte) {
|
| 990 | if (clear_flush_young_ptes_notify(vma, address, pvmw.pte, nr))
|
| 991 | referenced++;
|
| 992 | } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
|
| 993 | if (pmdp_clear_flush_young_notify(vma, address,
|
| 994 | pvmw.pmd))
|
| 995 | referenced++;
|
| 996 | } else {
|
| 997 |
|
| 998 | WARN_ON_ONCE(1);
|
| 999 | }
|
| 1000 |
|
| 1001 | ptes += nr;
|
| 1002 | pra->mapcount -= nr;
|
| 1003 | |
| 1004 | * If we are sure that we batched the entire folio,
|
| 1005 | * we can just optimize and stop right here.
|
| 1006 |
|
| 1007 | if (ptes == pvmw.nr_pages) {
|
| 1008 | page_vma_mapped_walk_done(&pvmw);
|
| 1009 | break;
|
| 1010 | }
|
| 1011 |
|
| 1012 |
|
| 1013 | pvmw.pte += nr - 1;
|
| 1014 | pvmw.address += (nr - 1) * PAGE_SIZE;
|
| 1015 | }
|
| 1016 |
|
| 1017 | if (referenced)
|
| 1018 | folio_clear_idle(folio);
|
| 1019 | if (folio_test_clear_young(folio))
|
| 1020 | referenced++;
|
| 1021 |
|
| 1022 | if (referenced) {
|
| 1023 | vma_flags_t vma_flags = vma->flags;
|
| 1024 |
|
| 1025 | pra->referenced++;
|
| 1026 | vma_flags_clear(&vma_flags, VMA_LOCKED_BIT);
|
| 1027 | vma_flags_set_mask(&pra->vma_flags, vma_flags);
|
| 1028 | }
|
| 1029 |
|
| 1030 | if (!pra->mapcount)
|
| 1031 | return false;
|
| 1032 |
|
| 1033 | return true;
|
| 1034 | }
|
| 1035 |
|
| 1036 | static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
|
| 1037 | {
|
| 1038 | struct folio_referenced_arg *pra = arg;
|
| 1039 | struct mem_cgroup *memcg = pra->memcg;
|
| 1040 |
|
| 1041 | |
| 1042 | * Ignore references from this mapping if it has no recency. If the
|
| 1043 | * folio has been used in another mapping, we will catch it; if this
|
| 1044 | * other mapping is already gone, the unmap path will have set the
|
| 1045 | * referenced flag or activated the folio in zap_pte_range().
|
| 1046 |
|
| 1047 | if (!vma_has_recency(vma))
|
| 1048 | return true;
|
| 1049 |
|
| 1050 | |
| 1051 | * If we are reclaiming on behalf of a cgroup, skip counting on behalf
|
| 1052 | * of references from different cgroups.
|
| 1053 |
|
| 1054 | if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
|
| 1055 | return true;
|
| 1056 |
|
| 1057 | return false;
|
| 1058 | }
|
| 1059 |
|
| 1060 | |
| 1061 | * folio_referenced() - Test if the folio was referenced.
|
| 1062 | * @folio: The folio to test.
|
| 1063 | * @is_locked: Caller holds lock on the folio.
|
| 1064 | * @memcg: target memory cgroup
|
| 1065 | * @vma_flags: A combination of all the vma->flags which referenced the folio.
|
| 1066 | *
|
| 1067 | * Quick test_and_clear_referenced for all mappings of a folio,
|
| 1068 | *
|
| 1069 | * Return: The number of mappings which referenced the folio. Return -1 if
|
| 1070 | * the function bailed out due to rmap lock contention.
|
| 1071 |
|
| 1072 | int folio_referenced(struct folio *folio, int is_locked,
|
| 1073 | struct mem_cgroup *memcg, vma_flags_t *vma_flags)
|
| 1074 | {
|
| 1075 | bool we_locked = false;
|
| 1076 | struct folio_referenced_arg pra = {
|
| 1077 | .mapcount = folio_mapcount(folio),
|
| 1078 | .memcg = memcg,
|
| 1079 | };
|
| 1080 | struct rmap_walk_control rwc = {
|
| 1081 | .rmap_one = folio_referenced_one,
|
| 1082 | .arg = (void *)&pra,
|
| 1083 | .anon_lock = folio_lock_anon_vma_read,
|
| 1084 | .try_lock = true,
|
| 1085 | .invalid_vma = invalid_folio_referenced_vma,
|
| 1086 | };
|
| 1087 |
|
| 1088 | VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio);
|
| 1089 | vma_flags_clear_all(vma_flags);
|
| 1090 | if (!pra.mapcount)
|
| 1091 | return 0;
|
| 1092 |
|
| 1093 | if (!folio_raw_mapping(folio))
|
| 1094 | return 0;
|
| 1095 |
|
| 1096 | if (!is_locked) {
|
| 1097 | we_locked = folio_trylock(folio);
|
| 1098 | if (!we_locked)
|
| 1099 | return 1;
|
| 1100 | }
|
| 1101 |
|
| 1102 | rmap_walk(folio, &rwc);
|
| 1103 | vma_flags_set_mask(vma_flags, pra.vma_flags);
|
| 1104 |
|
| 1105 | if (we_locked)
|
| 1106 | folio_unlock(folio);
|
| 1107 |
|
| 1108 | return rwc.contended ? -1 : pra.referenced;
|
| 1109 | }
|
| 1110 |
|
| 1111 | static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw)
|
| 1112 | {
|
| 1113 | int cleaned = 0;
|
| 1114 | struct vm_area_struct *vma = pvmw->vma;
|
| 1115 | struct mmu_notifier_range range;
|
| 1116 | unsigned long address = pvmw->address;
|
| 1117 |
|
| 1118 | |
| 1119 | * We have to assume the worse case ie pmd for invalidation. Note that
|
| 1120 | * the folio can not be freed from this function.
|
| 1121 |
|
| 1122 | mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
|
| 1123 | vma->vm_mm, address, vma_address_end(pvmw));
|
| 1124 | mmu_notifier_invalidate_range_start(&range);
|
| 1125 |
|
| 1126 | while (page_vma_mapped_walk(pvmw)) {
|
| 1127 | int ret = 0;
|
| 1128 |
|
| 1129 | address = pvmw->address;
|
| 1130 | if (pvmw->pte) {
|
| 1131 | pte_t *pte = pvmw->pte;
|
| 1132 | pte_t entry = ptep_get(pte);
|
| 1133 |
|
| 1134 | |
| 1135 | * PFN swap PTEs, such as device-exclusive ones, that
|
| 1136 | * actually map pages are clean and not writable from a
|
| 1137 | * CPU perspective. The MMU notifier takes care of any
|
| 1138 | * device aspects.
|
| 1139 |
|
| 1140 | if (!pte_present(entry))
|
| 1141 | continue;
|
| 1142 | if (!pte_dirty(entry) && !pte_write(entry))
|
| 1143 | continue;
|
| 1144 |
|
| 1145 | flush_cache_page(vma, address, pte_pfn(entry));
|
| 1146 | entry = ptep_clear_flush(vma, address, pte);
|
| 1147 | entry = pte_wrprotect(entry);
|
| 1148 | entry = pte_mkclean(entry);
|
| 1149 | set_pte_at(vma->vm_mm, address, pte, entry);
|
| 1150 | ret = 1;
|
| 1151 | } else {
|
| 1152 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 1153 | pmd_t *pmd = pvmw->pmd;
|
| 1154 | pmd_t entry = pmdp_get(pmd);
|
| 1155 |
|
| 1156 | |
| 1157 | * Please see the comment above (!pte_present).
|
| 1158 | * A non present PMD is not writable from a CPU
|
| 1159 | * perspective.
|
| 1160 |
|
| 1161 | if (!pmd_present(entry))
|
| 1162 | continue;
|
| 1163 | if (!pmd_dirty(entry) && !pmd_write(entry))
|
| 1164 | continue;
|
| 1165 |
|
| 1166 | flush_cache_range(vma, address,
|
| 1167 | address + HPAGE_PMD_SIZE);
|
| 1168 | entry = pmdp_invalidate(vma, address, pmd);
|
| 1169 | entry = pmd_wrprotect(entry);
|
| 1170 | entry = pmd_mkclean(entry);
|
| 1171 | set_pmd_at(vma->vm_mm, address, pmd, entry);
|
| 1172 | ret = 1;
|
| 1173 | #else
|
| 1174 |
|
| 1175 | WARN_ON_ONCE(1);
|
| 1176 | #endif
|
| 1177 | }
|
| 1178 |
|
| 1179 | if (ret)
|
| 1180 | cleaned++;
|
| 1181 | }
|
| 1182 |
|
| 1183 | mmu_notifier_invalidate_range_end(&range);
|
| 1184 |
|
| 1185 | return cleaned;
|
| 1186 | }
|
| 1187 |
|
| 1188 | static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma,
|
| 1189 | unsigned long address, void *arg)
|
| 1190 | {
|
| 1191 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC);
|
| 1192 | int *cleaned = arg;
|
| 1193 |
|
| 1194 | *cleaned += page_vma_mkclean_one(&pvmw);
|
| 1195 |
|
| 1196 | return true;
|
| 1197 | }
|
| 1198 |
|
| 1199 | static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
|
| 1200 | {
|
| 1201 | if (vma->vm_flags & VM_SHARED)
|
| 1202 | return false;
|
| 1203 |
|
| 1204 | return true;
|
| 1205 | }
|
| 1206 |
|
| 1207 | int folio_mkclean(struct folio *folio)
|
| 1208 | {
|
| 1209 | int cleaned = 0;
|
| 1210 | struct address_space *mapping;
|
| 1211 | struct rmap_walk_control rwc = {
|
| 1212 | .arg = (void *)&cleaned,
|
| 1213 | .rmap_one = page_mkclean_one,
|
| 1214 | .invalid_vma = invalid_mkclean_vma,
|
| 1215 | };
|
| 1216 |
|
| 1217 | BUG_ON(!folio_test_locked(folio));
|
| 1218 |
|
| 1219 | if (!folio_mapped(folio))
|
| 1220 | return 0;
|
| 1221 |
|
| 1222 | mapping = folio_mapping(folio);
|
| 1223 | if (!mapping)
|
| 1224 | return 0;
|
| 1225 |
|
| 1226 | rmap_walk(folio, &rwc);
|
| 1227 |
|
| 1228 | return cleaned;
|
| 1229 | }
|
| 1230 | EXPORT_SYMBOL_GPL(folio_mkclean);
|
| 1231 |
|
| 1232 | struct wrprotect_file_state {
|
| 1233 | int cleaned;
|
| 1234 | pgoff_t pgoff;
|
| 1235 | unsigned long pfn;
|
| 1236 | unsigned long nr_pages;
|
| 1237 | };
|
| 1238 |
|
| 1239 | static bool mapping_wrprotect_range_one(struct folio *folio,
|
| 1240 | struct vm_area_struct *vma, unsigned long address, void *arg)
|
| 1241 | {
|
| 1242 | struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg;
|
| 1243 | struct page_vma_mapped_walk pvmw = {
|
| 1244 | .pfn = state->pfn,
|
| 1245 | .nr_pages = state->nr_pages,
|
| 1246 | .pgoff = state->pgoff,
|
| 1247 | .vma = vma,
|
| 1248 | .address = address,
|
| 1249 | .flags = PVMW_SYNC,
|
| 1250 | .pgoff_is_anon = false,
|
| 1251 | };
|
| 1252 |
|
| 1253 | state->cleaned += page_vma_mkclean_one(&pvmw);
|
| 1254 |
|
| 1255 | return true;
|
| 1256 | }
|
| 1257 |
|
| 1258 | static void __rmap_walk_file(struct folio *folio, struct address_space *mapping,
|
| 1259 | pgoff_t pgoff_start, unsigned long nr_pages,
|
| 1260 | struct rmap_walk_control *rwc, bool locked);
|
| 1261 |
|
| 1262 | |
| 1263 | * mapping_wrprotect_range() - Write-protect all mappings in a specified range.
|
| 1264 | *
|
| 1265 | * @mapping: The mapping whose reverse mapping should be traversed.
|
| 1266 | * @pgoff: The page offset at which @pfn is mapped within @mapping.
|
| 1267 | * @pfn: The PFN of the page mapped in @mapping at @pgoff.
|
| 1268 | * @nr_pages: The number of physically contiguous base pages spanned.
|
| 1269 | *
|
| 1270 | * Traverses the reverse mapping, finding all VMAs which contain a shared
|
| 1271 | * mapping of the pages in the specified range in @mapping, and write-protects
|
| 1272 | * them (that is, updates the page tables to mark the mappings read-only such
|
| 1273 | * that a write protection fault arises when the mappings are written to).
|
| 1274 | *
|
| 1275 | * The @pfn value need not refer to a folio, but rather can reference a kernel
|
| 1276 | * allocation which is mapped into userland. We therefore do not require that
|
| 1277 | * the page maps to a folio with a valid mapping or index field, rather the
|
| 1278 | * caller specifies these in @mapping and @pgoff.
|
| 1279 | *
|
| 1280 | * Return: the number of write-protected PTEs, or an error.
|
| 1281 |
|
| 1282 | int mapping_wrprotect_range(struct address_space *mapping, pgoff_t pgoff,
|
| 1283 | unsigned long pfn, unsigned long nr_pages)
|
| 1284 | {
|
| 1285 | struct wrprotect_file_state state = {
|
| 1286 | .cleaned = 0,
|
| 1287 | .pgoff = pgoff,
|
| 1288 | .pfn = pfn,
|
| 1289 | .nr_pages = nr_pages,
|
| 1290 | };
|
| 1291 | struct rmap_walk_control rwc = {
|
| 1292 | .arg = (void *)&state,
|
| 1293 | .rmap_one = mapping_wrprotect_range_one,
|
| 1294 | .invalid_vma = invalid_mkclean_vma,
|
| 1295 | };
|
| 1296 |
|
| 1297 | if (!mapping)
|
| 1298 | return 0;
|
| 1299 |
|
| 1300 | __rmap_walk_file(NULL, mapping, pgoff, nr_pages, &rwc,
|
| 1301 | false);
|
| 1302 |
|
| 1303 | return state.cleaned;
|
| 1304 | }
|
| 1305 | EXPORT_SYMBOL_GPL(mapping_wrprotect_range);
|
| 1306 |
|
| 1307 | |
| 1308 | * pfn_mkclean_range - Cleans the PTEs (including PMDs) mapped with range of
|
| 1309 | * [@pfn, @pfn + @nr_pages) at the specific offset (@pgoff)
|
| 1310 | * within the @vma of shared mappings. And since clean PTEs
|
| 1311 | * should also be readonly, write protects them too.
|
| 1312 | * @pfn: start pfn.
|
| 1313 | * @nr_pages: number of physically contiguous pages srarting with @pfn.
|
| 1314 | * @pgoff: page offset that the @pfn mapped with.
|
| 1315 | * @vma: vma that @pfn mapped within.
|
| 1316 | *
|
| 1317 | * Returns the number of cleaned PTEs (including PMDs).
|
| 1318 |
|
| 1319 | int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
|
| 1320 | struct vm_area_struct *vma)
|
| 1321 | {
|
| 1322 | struct page_vma_mapped_walk pvmw = {
|
| 1323 | .pfn = pfn,
|
| 1324 | .nr_pages = nr_pages,
|
| 1325 | .pgoff = pgoff,
|
| 1326 | .vma = vma,
|
| 1327 | .flags = PVMW_SYNC,
|
| 1328 | .pgoff_is_anon = false,
|
| 1329 | };
|
| 1330 |
|
| 1331 | if (invalid_mkclean_vma(vma, NULL))
|
| 1332 | return 0;
|
| 1333 |
|
| 1334 | pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages);
|
| 1335 | VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);
|
| 1336 |
|
| 1337 | return page_vma_mkclean_one(&pvmw);
|
| 1338 | }
|
| 1339 |
|
| 1340 | static void __folio_mod_stat(struct folio *folio, int nr, int nr_pmdmapped)
|
| 1341 | {
|
| 1342 | int idx;
|
| 1343 |
|
| 1344 | if (nr) {
|
| 1345 | idx = folio_test_anon(folio) ? NR_ANON_MAPPED : NR_FILE_MAPPED;
|
| 1346 | lruvec_stat_mod_folio(folio, idx, nr);
|
| 1347 | }
|
| 1348 | if (nr_pmdmapped) {
|
| 1349 | if (folio_test_anon(folio)) {
|
| 1350 | idx = NR_ANON_THPS;
|
| 1351 | lruvec_stat_mod_folio(folio, idx, nr_pmdmapped);
|
| 1352 | } else {
|
| 1353 |
|
| 1354 | idx = folio_test_swapbacked(folio) ?
|
| 1355 | NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED;
|
| 1356 | __mod_node_page_state(folio_pgdat(folio), idx,
|
| 1357 | nr_pmdmapped);
|
| 1358 | }
|
| 1359 | }
|
| 1360 | }
|
| 1361 |
|
| 1362 | static __always_inline void __folio_add_rmap(struct folio *folio,
|
| 1363 | struct page *page, int nr_pages, struct vm_area_struct *vma,
|
| 1364 | enum pgtable_level level)
|
| 1365 | {
|
| 1366 | atomic_t *mapped = &folio->_nr_pages_mapped;
|
| 1367 | const int orig_nr_pages = nr_pages;
|
| 1368 | int first = 0, nr = 0, nr_pmdmapped = 0;
|
| 1369 |
|
| 1370 | __folio_rmap_sanity_checks(folio, page, nr_pages, level);
|
| 1371 |
|
| 1372 | switch (level) {
|
| 1373 | case PGTABLE_LEVEL_PTE:
|
| 1374 | if (!folio_test_large(folio)) {
|
| 1375 | nr = atomic_inc_and_test(&folio->_mapcount);
|
| 1376 | break;
|
| 1377 | }
|
| 1378 |
|
| 1379 | if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
|
| 1380 | nr = folio_add_return_large_mapcount(folio, orig_nr_pages, vma);
|
| 1381 | if (nr == orig_nr_pages)
|
| 1382 |
|
| 1383 | nr = folio_large_nr_pages(folio);
|
| 1384 | else
|
| 1385 | nr = 0;
|
| 1386 | break;
|
| 1387 | }
|
| 1388 |
|
| 1389 | do {
|
| 1390 | first += atomic_inc_and_test(&page->_mapcount);
|
| 1391 | } while (page++, --nr_pages > 0);
|
| 1392 |
|
| 1393 | if (first &&
|
| 1394 | atomic_add_return_relaxed(first, mapped) < ENTIRELY_MAPPED)
|
| 1395 | nr = first;
|
| 1396 |
|
| 1397 | folio_add_large_mapcount(folio, orig_nr_pages, vma);
|
| 1398 | break;
|
| 1399 | case PGTABLE_LEVEL_PMD:
|
| 1400 | case PGTABLE_LEVEL_PUD:
|
| 1401 | first = atomic_inc_and_test(&folio->_entire_mapcount);
|
| 1402 | if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
|
| 1403 | if (level == PGTABLE_LEVEL_PMD && first)
|
| 1404 | nr_pmdmapped = folio_large_nr_pages(folio);
|
| 1405 | nr = folio_inc_return_large_mapcount(folio, vma);
|
| 1406 | if (nr == 1)
|
| 1407 |
|
| 1408 | nr = folio_large_nr_pages(folio);
|
| 1409 | else
|
| 1410 | nr = 0;
|
| 1411 | break;
|
| 1412 | }
|
| 1413 |
|
| 1414 | if (first) {
|
| 1415 | nr = atomic_add_return_relaxed(ENTIRELY_MAPPED, mapped);
|
| 1416 | if (likely(nr < ENTIRELY_MAPPED + ENTIRELY_MAPPED)) {
|
| 1417 | nr_pages = folio_large_nr_pages(folio);
|
| 1418 | |
| 1419 | * We only track PMD mappings of PMD-sized
|
| 1420 | * folios separately.
|
| 1421 |
|
| 1422 | if (level == PGTABLE_LEVEL_PMD)
|
| 1423 | nr_pmdmapped = nr_pages;
|
| 1424 | nr = nr_pages - (nr & FOLIO_PAGES_MAPPED);
|
| 1425 |
|
| 1426 | if (unlikely(nr < 0))
|
| 1427 | nr = 0;
|
| 1428 | } else {
|
| 1429 |
|
| 1430 | nr = 0;
|
| 1431 | }
|
| 1432 | }
|
| 1433 | folio_inc_large_mapcount(folio, vma);
|
| 1434 | break;
|
| 1435 | default:
|
| 1436 | BUILD_BUG();
|
| 1437 | }
|
| 1438 | __folio_mod_stat(folio, nr, nr_pmdmapped);
|
| 1439 | }
|
| 1440 |
|
| 1441 | |
| 1442 | * folio_move_anon_rmap - move a folio to our anon_vma
|
| 1443 | * @folio: The folio to move to our anon_vma
|
| 1444 | * @vma: The vma the folio belongs to
|
| 1445 | *
|
| 1446 | * When a folio belongs exclusively to one process after a COW event,
|
| 1447 | * that folio can be moved into the anon_vma that belongs to just that
|
| 1448 | * process, so the rmap code will not search the parent or sibling processes.
|
| 1449 |
|
| 1450 | void folio_move_anon_rmap(struct folio *folio, struct vm_area_struct *vma)
|
| 1451 | {
|
| 1452 | void *anon_vma = vma->anon_vma;
|
| 1453 |
|
| 1454 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 1455 | VM_BUG_ON_VMA(!anon_vma, vma);
|
| 1456 |
|
| 1457 | anon_vma += FOLIO_MAPPING_ANON;
|
| 1458 | |
| 1459 | * Ensure that anon_vma and the FOLIO_MAPPING_ANON bit are written
|
| 1460 | * simultaneously, so a concurrent reader (eg folio_referenced()'s
|
| 1461 | * folio_test_anon()) will not see one without the other.
|
| 1462 |
|
| 1463 | WRITE_ONCE(folio->mapping, anon_vma);
|
| 1464 | }
|
| 1465 |
|
| 1466 | |
| 1467 | * __folio_set_anon - set up a new anonymous rmap for a folio
|
| 1468 | * @folio: The folio to set up the new anonymous rmap for.
|
| 1469 | * @vma: VM area to add the folio to.
|
| 1470 | * @address: User virtual address of the mapping
|
| 1471 | * @exclusive: Whether the folio is exclusive to the process.
|
| 1472 |
|
| 1473 | static void __folio_set_anon(struct folio *folio, struct vm_area_struct *vma,
|
| 1474 | unsigned long address, bool exclusive)
|
| 1475 | {
|
| 1476 | struct anon_vma *anon_vma = vma->anon_vma;
|
| 1477 |
|
| 1478 | BUG_ON(!anon_vma);
|
| 1479 |
|
| 1480 | |
| 1481 | * If the folio isn't exclusive to this vma, we must use the _oldest_
|
| 1482 | * possible anon_vma for the folio mapping!
|
| 1483 |
|
| 1484 | if (!exclusive)
|
| 1485 | anon_vma = anon_vma->root;
|
| 1486 |
|
| 1487 | |
| 1488 | * page_idle does a lockless/optimistic rmap scan on folio->mapping.
|
| 1489 | * Make sure the compiler doesn't split the stores of anon_vma and
|
| 1490 | * the FOLIO_MAPPING_ANON type identifier, otherwise the rmap code
|
| 1491 | * could mistake the mapping for a struct address_space and crash.
|
| 1492 |
|
| 1493 | anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON;
|
| 1494 | WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
|
| 1495 | folio->index = linear_anon_page_index(vma, address);
|
| 1496 | }
|
| 1497 |
|
| 1498 | |
| 1499 | * __page_check_anon_rmap - sanity check anonymous rmap addition
|
| 1500 | * @folio: The folio containing @page.
|
| 1501 | * @page: the page to check the mapping of
|
| 1502 | * @vma: the vm area in which the mapping is added
|
| 1503 | * @address: the user virtual address mapped
|
| 1504 |
|
| 1505 | static void __page_check_anon_rmap(const struct folio *folio,
|
| 1506 | const struct page *page, struct vm_area_struct *vma,
|
| 1507 | unsigned long address)
|
| 1508 | {
|
| 1509 | |
| 1510 | * The page's anon-rmap details (mapping and index) are guaranteed to
|
| 1511 | * be set up correctly at this point.
|
| 1512 | *
|
| 1513 | * We have exclusion against folio_add_anon_rmap_*() because the caller
|
| 1514 | * always holds the page locked.
|
| 1515 | *
|
| 1516 | * We have exclusion against folio_add_new_anon_rmap because those pages
|
| 1517 | * are initially only visible via the pagetables, and the pte is locked
|
| 1518 | * over the call to folio_add_new_anon_rmap.
|
| 1519 |
|
| 1520 | VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
|
| 1521 | folio);
|
| 1522 | VM_BUG_ON_PAGE(page_pgoff(folio, page) !=
|
| 1523 | linear_anon_page_index(vma, address), page);
|
| 1524 | }
|
| 1525 |
|
| 1526 | static __always_inline void __folio_add_anon_rmap(struct folio *folio,
|
| 1527 | struct page *page, int nr_pages, struct vm_area_struct *vma,
|
| 1528 | unsigned long address, rmap_t flags, enum pgtable_level level)
|
| 1529 | {
|
| 1530 | int i;
|
| 1531 |
|
| 1532 | VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
|
| 1533 |
|
| 1534 | __folio_add_rmap(folio, page, nr_pages, vma, level);
|
| 1535 |
|
| 1536 | if (likely(!folio_test_ksm(folio)))
|
| 1537 | __page_check_anon_rmap(folio, page, vma, address);
|
| 1538 |
|
| 1539 | if (flags & RMAP_EXCLUSIVE) {
|
| 1540 | switch (level) {
|
| 1541 | case PGTABLE_LEVEL_PTE:
|
| 1542 | for (i = 0; i < nr_pages; i++)
|
| 1543 | SetPageAnonExclusive(page + i);
|
| 1544 | break;
|
| 1545 | case PGTABLE_LEVEL_PMD:
|
| 1546 | SetPageAnonExclusive(page);
|
| 1547 | break;
|
| 1548 | case PGTABLE_LEVEL_PUD:
|
| 1549 | |
| 1550 | * Keep the compiler happy, we don't support anonymous
|
| 1551 | * PUD mappings.
|
| 1552 |
|
| 1553 | WARN_ON_ONCE(1);
|
| 1554 | break;
|
| 1555 | default:
|
| 1556 | BUILD_BUG();
|
| 1557 | }
|
| 1558 | }
|
| 1559 |
|
| 1560 | VM_WARN_ON_FOLIO(!folio_test_large(folio) && PageAnonExclusive(page) &&
|
| 1561 | atomic_read(&folio->_mapcount) > 0, folio);
|
| 1562 | for (i = 0; i < nr_pages; i++) {
|
| 1563 | struct page *cur_page = page + i;
|
| 1564 |
|
| 1565 | VM_WARN_ON_FOLIO(folio_test_large(folio) &&
|
| 1566 | folio_entire_mapcount(folio) > 1 &&
|
| 1567 | PageAnonExclusive(cur_page), folio);
|
| 1568 | if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT))
|
| 1569 | continue;
|
| 1570 |
|
| 1571 | |
| 1572 | * While PTE-mapping a THP we have a PMD and a PTE
|
| 1573 | * mapping.
|
| 1574 |
|
| 1575 | VM_WARN_ON_FOLIO(atomic_read(&cur_page->_mapcount) > 0 &&
|
| 1576 | PageAnonExclusive(cur_page), folio);
|
| 1577 | }
|
| 1578 |
|
| 1579 | |
| 1580 | * Only mlock it if the folio is fully mapped to the VMA.
|
| 1581 | *
|
| 1582 | * Partially mapped folios can be split on reclaim and part outside
|
| 1583 | * of mlocked VMA can be evicted or freed.
|
| 1584 |
|
| 1585 | if (folio_nr_pages(folio) == nr_pages)
|
| 1586 | mlock_vma_folio(folio, vma);
|
| 1587 | }
|
| 1588 |
|
| 1589 | |
| 1590 | * folio_add_anon_rmap_ptes - add PTE mappings to a page range of an anon folio
|
| 1591 | * @folio: The folio to add the mappings to
|
| 1592 | * @page: The first page to add
|
| 1593 | * @nr_pages: The number of pages which will be mapped
|
| 1594 | * @vma: The vm area in which the mappings are added
|
| 1595 | * @address: The user virtual address of the first page to map
|
| 1596 | * @flags: The rmap flags
|
| 1597 | *
|
| 1598 | * The page range of folio is defined by [first_page, first_page + nr_pages)
|
| 1599 | *
|
| 1600 | * The caller needs to hold the page table lock, and the page must be locked in
|
| 1601 | * the anon_vma case: to serialize mapping,index checking after setting,
|
| 1602 | * and to ensure that an anon folio is not being upgraded racily to a KSM folio
|
| 1603 | * (but KSM folios are never downgraded).
|
| 1604 |
|
| 1605 | void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page,
|
| 1606 | int nr_pages, struct vm_area_struct *vma, unsigned long address,
|
| 1607 | rmap_t flags)
|
| 1608 | {
|
| 1609 | __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags,
|
| 1610 | PGTABLE_LEVEL_PTE);
|
| 1611 | }
|
| 1612 |
|
| 1613 | |
| 1614 | * folio_add_anon_rmap_pmd - add a PMD mapping to a page range of an anon folio
|
| 1615 | * @folio: The folio to add the mapping to
|
| 1616 | * @page: The first page to add
|
| 1617 | * @vma: The vm area in which the mapping is added
|
| 1618 | * @address: The user virtual address of the first page to map
|
| 1619 | * @flags: The rmap flags
|
| 1620 | *
|
| 1621 | * The page range of folio is defined by [first_page, first_page + HPAGE_PMD_NR)
|
| 1622 | *
|
| 1623 | * The caller needs to hold the page table lock, and the page must be locked in
|
| 1624 | * the anon_vma case: to serialize mapping,index checking after setting.
|
| 1625 |
|
| 1626 | void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page,
|
| 1627 | struct vm_area_struct *vma, unsigned long address, rmap_t flags)
|
| 1628 | {
|
| 1629 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 1630 | __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags,
|
| 1631 | PGTABLE_LEVEL_PMD);
|
| 1632 | #else
|
| 1633 | WARN_ON_ONCE(true);
|
| 1634 | #endif
|
| 1635 | }
|
| 1636 |
|
| 1637 | |
| 1638 | * folio_add_new_anon_rmap - Add mapping to a new anonymous folio.
|
| 1639 | * @folio: The folio to add the mapping to.
|
| 1640 | * @vma: the vm area in which the mapping is added
|
| 1641 | * @address: the user virtual address mapped
|
| 1642 | * @flags: The rmap flags
|
| 1643 | *
|
| 1644 | * Like folio_add_anon_rmap_*() but must only be called on *new* folios.
|
| 1645 | * This means the inc-and-test can be bypassed.
|
| 1646 | * The folio doesn't necessarily need to be locked while it's exclusive
|
| 1647 | * unless two threads map it concurrently. However, the folio must be
|
| 1648 | * locked if it's shared.
|
| 1649 | *
|
| 1650 | * If the folio is pmd-mappable, it is accounted as a THP.
|
| 1651 |
|
| 1652 | void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
|
| 1653 | unsigned long address, rmap_t flags)
|
| 1654 | {
|
| 1655 | const bool exclusive = flags & RMAP_EXCLUSIVE;
|
| 1656 | int nr = 1, nr_pmdmapped = 0;
|
| 1657 |
|
| 1658 | VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
|
| 1659 | VM_WARN_ON_FOLIO(!exclusive && !folio_test_locked(folio), folio);
|
| 1660 |
|
| 1661 | |
| 1662 | * VM_DROPPABLE mappings don't swap; instead they're just dropped when
|
| 1663 | * under memory pressure.
|
| 1664 |
|
| 1665 | if (!folio_test_swapbacked(folio) && !(vma->vm_flags & VM_DROPPABLE))
|
| 1666 | __folio_set_swapbacked(folio);
|
| 1667 | __folio_set_anon(folio, vma, address, exclusive);
|
| 1668 |
|
| 1669 | if (likely(!folio_test_large(folio))) {
|
| 1670 |
|
| 1671 | atomic_set(&folio->_mapcount, 0);
|
| 1672 | if (exclusive)
|
| 1673 | SetPageAnonExclusive(&folio->page);
|
| 1674 | } else if (!folio_test_pmd_mappable(folio)) {
|
| 1675 | int i;
|
| 1676 |
|
| 1677 | nr = folio_large_nr_pages(folio);
|
| 1678 | for (i = 0; i < nr; i++) {
|
| 1679 | struct page *page = folio_page(folio, i);
|
| 1680 |
|
| 1681 | if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
|
| 1682 |
|
| 1683 | atomic_set(&page->_mapcount, 0);
|
| 1684 | if (exclusive)
|
| 1685 | SetPageAnonExclusive(page);
|
| 1686 | }
|
| 1687 |
|
| 1688 | folio_set_large_mapcount(folio, nr, vma);
|
| 1689 | if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
|
| 1690 | atomic_set(&folio->_nr_pages_mapped, nr);
|
| 1691 | } else {
|
| 1692 | nr = folio_large_nr_pages(folio);
|
| 1693 |
|
| 1694 | atomic_set(&folio->_entire_mapcount, 0);
|
| 1695 | folio_set_large_mapcount(folio, 1, vma);
|
| 1696 | if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
|
| 1697 | atomic_set(&folio->_nr_pages_mapped, ENTIRELY_MAPPED);
|
| 1698 | if (exclusive)
|
| 1699 | SetPageAnonExclusive(&folio->page);
|
| 1700 | nr_pmdmapped = nr;
|
| 1701 | }
|
| 1702 |
|
| 1703 | VM_WARN_ON_ONCE(address < vma->vm_start ||
|
| 1704 | address + (nr << PAGE_SHIFT) > vma->vm_end);
|
| 1705 |
|
| 1706 | __folio_mod_stat(folio, nr, nr_pmdmapped);
|
| 1707 | mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
|
| 1708 | }
|
| 1709 |
|
| 1710 | static __always_inline void __folio_add_file_rmap(struct folio *folio,
|
| 1711 | struct page *page, int nr_pages, struct vm_area_struct *vma,
|
| 1712 | enum pgtable_level level)
|
| 1713 | {
|
| 1714 | VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);
|
| 1715 |
|
| 1716 | __folio_add_rmap(folio, page, nr_pages, vma, level);
|
| 1717 |
|
| 1718 | |
| 1719 | * Only mlock it if the folio is fully mapped to the VMA.
|
| 1720 | *
|
| 1721 | * Partially mapped folios can be split on reclaim and part outside
|
| 1722 | * of mlocked VMA can be evicted or freed.
|
| 1723 |
|
| 1724 | if (folio_nr_pages(folio) == nr_pages)
|
| 1725 | mlock_vma_folio(folio, vma);
|
| 1726 | }
|
| 1727 |
|
| 1728 | |
| 1729 | * folio_add_file_rmap_ptes - add PTE mappings to a page range of a folio
|
| 1730 | * @folio: The folio to add the mappings to
|
| 1731 | * @page: The first page to add
|
| 1732 | * @nr_pages: The number of pages that will be mapped using PTEs
|
| 1733 | * @vma: The vm area in which the mappings are added
|
| 1734 | *
|
| 1735 | * The page range of the folio is defined by [page, page + nr_pages)
|
| 1736 | *
|
| 1737 | * The caller needs to hold the page table lock.
|
| 1738 |
|
| 1739 | void folio_add_file_rmap_ptes(struct folio *folio, struct page *page,
|
| 1740 | int nr_pages, struct vm_area_struct *vma)
|
| 1741 | {
|
| 1742 | __folio_add_file_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE);
|
| 1743 | }
|
| 1744 |
|
| 1745 | |
| 1746 | * folio_add_file_rmap_pmd - add a PMD mapping to a page range of a folio
|
| 1747 | * @folio: The folio to add the mapping to
|
| 1748 | * @page: The first page to add
|
| 1749 | * @vma: The vm area in which the mapping is added
|
| 1750 | *
|
| 1751 | * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
|
| 1752 | *
|
| 1753 | * The caller needs to hold the page table lock.
|
| 1754 |
|
| 1755 | void folio_add_file_rmap_pmd(struct folio *folio, struct page *page,
|
| 1756 | struct vm_area_struct *vma)
|
| 1757 | {
|
| 1758 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 1759 | __folio_add_file_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD);
|
| 1760 | #else
|
| 1761 | WARN_ON_ONCE(true);
|
| 1762 | #endif
|
| 1763 | }
|
| 1764 |
|
| 1765 | |
| 1766 | * folio_add_file_rmap_pud - add a PUD mapping to a page range of a folio
|
| 1767 | * @folio: The folio to add the mapping to
|
| 1768 | * @page: The first page to add
|
| 1769 | * @vma: The vm area in which the mapping is added
|
| 1770 | *
|
| 1771 | * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
|
| 1772 | *
|
| 1773 | * The caller needs to hold the page table lock.
|
| 1774 |
|
| 1775 | void folio_add_file_rmap_pud(struct folio *folio, struct page *page,
|
| 1776 | struct vm_area_struct *vma)
|
| 1777 | {
|
| 1778 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
|
| 1779 | defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
|
| 1780 | __folio_add_file_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD);
|
| 1781 | #else
|
| 1782 | WARN_ON_ONCE(true);
|
| 1783 | #endif
|
| 1784 | }
|
| 1785 |
|
| 1786 | static __always_inline void __folio_remove_rmap(struct folio *folio,
|
| 1787 | struct page *page, int nr_pages, struct vm_area_struct *vma,
|
| 1788 | enum pgtable_level level)
|
| 1789 | {
|
| 1790 | atomic_t *mapped = &folio->_nr_pages_mapped;
|
| 1791 | int last = 0, nr = 0, nr_pmdmapped = 0;
|
| 1792 | bool partially_mapped = false;
|
| 1793 |
|
| 1794 | __folio_rmap_sanity_checks(folio, page, nr_pages, level);
|
| 1795 |
|
| 1796 | switch (level) {
|
| 1797 | case PGTABLE_LEVEL_PTE:
|
| 1798 | if (!folio_test_large(folio)) {
|
| 1799 | nr = atomic_add_negative(-1, &folio->_mapcount);
|
| 1800 | break;
|
| 1801 | }
|
| 1802 |
|
| 1803 | if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
|
| 1804 | nr = folio_sub_return_large_mapcount(folio, nr_pages, vma);
|
| 1805 | if (!nr) {
|
| 1806 |
|
| 1807 | nr = folio_large_nr_pages(folio);
|
| 1808 | } else {
|
| 1809 | partially_mapped = nr < folio_large_nr_pages(folio) &&
|
| 1810 | !folio_entire_mapcount(folio);
|
| 1811 | nr = 0;
|
| 1812 | }
|
| 1813 | break;
|
| 1814 | }
|
| 1815 |
|
| 1816 | folio_sub_large_mapcount(folio, nr_pages, vma);
|
| 1817 | do {
|
| 1818 | last += atomic_add_negative(-1, &page->_mapcount);
|
| 1819 | } while (page++, --nr_pages > 0);
|
| 1820 |
|
| 1821 | if (last &&
|
| 1822 | atomic_sub_return_relaxed(last, mapped) < ENTIRELY_MAPPED)
|
| 1823 | nr = last;
|
| 1824 |
|
| 1825 | partially_mapped = nr && atomic_read(mapped);
|
| 1826 | break;
|
| 1827 | case PGTABLE_LEVEL_PMD:
|
| 1828 | case PGTABLE_LEVEL_PUD:
|
| 1829 | if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) {
|
| 1830 | last = atomic_add_negative(-1, &folio->_entire_mapcount);
|
| 1831 | if (level == PGTABLE_LEVEL_PMD && last)
|
| 1832 | nr_pmdmapped = folio_large_nr_pages(folio);
|
| 1833 | nr = folio_dec_return_large_mapcount(folio, vma);
|
| 1834 | if (!nr) {
|
| 1835 |
|
| 1836 | nr = folio_large_nr_pages(folio);
|
| 1837 | } else {
|
| 1838 | partially_mapped = last &&
|
| 1839 | nr < folio_large_nr_pages(folio);
|
| 1840 | nr = 0;
|
| 1841 | }
|
| 1842 | break;
|
| 1843 | }
|
| 1844 |
|
| 1845 | folio_dec_large_mapcount(folio, vma);
|
| 1846 | last = atomic_add_negative(-1, &folio->_entire_mapcount);
|
| 1847 | if (last) {
|
| 1848 | nr = atomic_sub_return_relaxed(ENTIRELY_MAPPED, mapped);
|
| 1849 | if (likely(nr < ENTIRELY_MAPPED)) {
|
| 1850 | nr_pages = folio_large_nr_pages(folio);
|
| 1851 | if (level == PGTABLE_LEVEL_PMD)
|
| 1852 | nr_pmdmapped = nr_pages;
|
| 1853 | nr = nr_pages - nr;
|
| 1854 |
|
| 1855 | if (unlikely(nr < 0))
|
| 1856 | nr = 0;
|
| 1857 | } else {
|
| 1858 |
|
| 1859 | nr = 0;
|
| 1860 | }
|
| 1861 | }
|
| 1862 |
|
| 1863 | partially_mapped = nr && nr < nr_pmdmapped;
|
| 1864 | break;
|
| 1865 | default:
|
| 1866 | BUILD_BUG();
|
| 1867 | }
|
| 1868 |
|
| 1869 | |
| 1870 | * Queue anon large folio for deferred split if at least one page of
|
| 1871 | * the folio is unmapped and at least one page is still mapped.
|
| 1872 | *
|
| 1873 | * Check partially_mapped first to ensure it is a large folio.
|
| 1874 | *
|
| 1875 | * Device private folios do not support deferred splitting and
|
| 1876 | * shrinker based scanning of the folios to free.
|
| 1877 |
|
| 1878 | if (partially_mapped && folio_test_anon(folio) &&
|
| 1879 | !folio_test_partially_mapped(folio) &&
|
| 1880 | !folio_is_device_private(folio))
|
| 1881 | deferred_split_folio(folio, true);
|
| 1882 |
|
| 1883 | __folio_mod_stat(folio, -nr, -nr_pmdmapped);
|
| 1884 |
|
| 1885 | |
| 1886 | * It would be tidy to reset folio_test_anon mapping when fully
|
| 1887 | * unmapped, but that might overwrite a racing folio_add_anon_rmap_*()
|
| 1888 | * which increments mapcount after us but sets mapping before us:
|
| 1889 | * so leave the reset to free_pages_prepare, and remember that
|
| 1890 | * it's only reliable while mapped.
|
| 1891 |
|
| 1892 |
|
| 1893 | munlock_vma_folio(folio, vma);
|
| 1894 | }
|
| 1895 |
|
| 1896 | |
| 1897 | * folio_remove_rmap_ptes - remove PTE mappings from a page range of a folio
|
| 1898 | * @folio: The folio to remove the mappings from
|
| 1899 | * @page: The first page to remove
|
| 1900 | * @nr_pages: The number of pages that will be removed from the mapping
|
| 1901 | * @vma: The vm area from which the mappings are removed
|
| 1902 | *
|
| 1903 | * The page range of the folio is defined by [page, page + nr_pages)
|
| 1904 | *
|
| 1905 | * The caller needs to hold the page table lock.
|
| 1906 |
|
| 1907 | void folio_remove_rmap_ptes(struct folio *folio, struct page *page,
|
| 1908 | int nr_pages, struct vm_area_struct *vma)
|
| 1909 | {
|
| 1910 | __folio_remove_rmap(folio, page, nr_pages, vma, PGTABLE_LEVEL_PTE);
|
| 1911 | }
|
| 1912 |
|
| 1913 | |
| 1914 | * folio_remove_rmap_pmd - remove a PMD mapping from a page range of a folio
|
| 1915 | * @folio: The folio to remove the mapping from
|
| 1916 | * @page: The first page to remove
|
| 1917 | * @vma: The vm area from which the mapping is removed
|
| 1918 | *
|
| 1919 | * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
|
| 1920 | *
|
| 1921 | * The caller needs to hold the page table lock.
|
| 1922 |
|
| 1923 | void folio_remove_rmap_pmd(struct folio *folio, struct page *page,
|
| 1924 | struct vm_area_struct *vma)
|
| 1925 | {
|
| 1926 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 1927 | __folio_remove_rmap(folio, page, HPAGE_PMD_NR, vma, PGTABLE_LEVEL_PMD);
|
| 1928 | #else
|
| 1929 | WARN_ON_ONCE(true);
|
| 1930 | #endif
|
| 1931 | }
|
| 1932 |
|
| 1933 | |
| 1934 | * folio_remove_rmap_pud - remove a PUD mapping from a page range of a folio
|
| 1935 | * @folio: The folio to remove the mapping from
|
| 1936 | * @page: The first page to remove
|
| 1937 | * @vma: The vm area from which the mapping is removed
|
| 1938 | *
|
| 1939 | * The page range of the folio is defined by [page, page + HPAGE_PUD_NR)
|
| 1940 | *
|
| 1941 | * The caller needs to hold the page table lock.
|
| 1942 |
|
| 1943 | void folio_remove_rmap_pud(struct folio *folio, struct page *page,
|
| 1944 | struct vm_area_struct *vma)
|
| 1945 | {
|
| 1946 | #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
|
| 1947 | defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
|
| 1948 | __folio_remove_rmap(folio, page, HPAGE_PUD_NR, vma, PGTABLE_LEVEL_PUD);
|
| 1949 | #else
|
| 1950 | WARN_ON_ONCE(true);
|
| 1951 | #endif
|
| 1952 | }
|
| 1953 |
|
| 1954 | static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
|
| 1955 | struct page_vma_mapped_walk *pvmw,
|
| 1956 | enum ttu_flags flags, pte_t pte)
|
| 1957 | {
|
| 1958 | unsigned long end_addr, addr = pvmw->address;
|
| 1959 | struct vm_area_struct *vma = pvmw->vma;
|
| 1960 | unsigned int max_nr;
|
| 1961 |
|
| 1962 | if (flags & TTU_HWPOISON)
|
| 1963 | return 1;
|
| 1964 | if (!folio_test_large(folio))
|
| 1965 | return 1;
|
| 1966 |
|
| 1967 |
|
| 1968 | end_addr = pmd_addr_end(addr, vma->vm_end);
|
| 1969 | max_nr = (end_addr - addr) >> PAGE_SHIFT;
|
| 1970 |
|
| 1971 |
|
| 1972 | if (folio_test_anon(folio) && folio_test_swapbacked(folio))
|
| 1973 | return 1;
|
| 1974 |
|
| 1975 | if (pte_unused(pte))
|
| 1976 | return 1;
|
| 1977 |
|
| 1978 | |
| 1979 | * If unmap fails, we need to restore the ptes. To avoid accidentally
|
| 1980 | * upgrading write permissions for ptes that were not originally
|
| 1981 | * writable, and to avoid losing the soft-dirty bit, use the
|
| 1982 | * appropriate FPB flags.
|
| 1983 |
|
| 1984 | return folio_pte_batch_flags(folio, vma, pvmw->pte, &pte, max_nr,
|
| 1985 | FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY);
|
| 1986 | }
|
| 1987 |
|
| 1988 | static bool try_to_unmap_poisoned_hugetlb_one(struct folio *folio,
|
| 1989 | struct vm_area_struct *vma, unsigned long address, void *arg)
|
| 1990 | {
|
| 1991 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
|
| 1992 | const unsigned long hsz = huge_page_size(hstate_vma(vma));
|
| 1993 | const enum ttu_flags flags = (enum ttu_flags)(long)arg;
|
| 1994 | struct mm_struct *mm = vma->vm_mm;
|
| 1995 | struct mmu_notifier_range range;
|
| 1996 | bool ret = true;
|
| 1997 | pte_t pteval;
|
| 1998 |
|
| 1999 | |
| 2000 | * The try_to_unmap() is only passed a hugetlb folio in the case
|
| 2001 | * where the hugetlb folio is poisoned.
|
| 2002 |
|
| 2003 | VM_WARN_ON_ONCE_FOLIO(!folio_test_hwpoison(folio), folio);
|
| 2004 | VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON));
|
| 2005 |
|
| 2006 | range.end = vma_address_end(&pvmw);
|
| 2007 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
| 2008 | address, range.end);
|
| 2009 | adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
|
| 2010 | mmu_notifier_invalidate_range_start(&range);
|
| 2011 |
|
| 2012 |
|
| 2013 | if (!page_vma_mapped_walk(&pvmw))
|
| 2014 | goto range_end;
|
| 2015 |
|
| 2016 | VM_WARN_ON_ONCE(address != pvmw.address);
|
| 2017 |
|
| 2018 | pteval = huge_ptep_get(mm, address, pvmw.pte);
|
| 2019 | VM_WARN_ON_ONCE(!pte_present(pteval));
|
| 2020 | VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio));
|
| 2021 |
|
| 2022 | |
| 2023 | * huge_pmd_unshare may unmap an entire PMD page. There is no way of
|
| 2024 | * knowing exactly which PMDs may be cached for this mm, so we must
|
| 2025 | * flush them all. start/end were already adjusted above to cover this
|
| 2026 | * range.
|
| 2027 |
|
| 2028 | flush_cache_range(vma, range.start, range.end);
|
| 2029 |
|
| 2030 | |
| 2031 | * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode.
|
| 2032 | * Caller needs to explicitly do this outside rmap routines.
|
| 2033 | *
|
| 2034 | * We also must hold hugetlb vma_lock in write mode. Lock order dictates
|
| 2035 | * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and
|
| 2036 | * fail if unsuccessful.
|
| 2037 |
|
| 2038 | if (!folio_test_anon(folio)) {
|
| 2039 | struct mmu_gather tlb;
|
| 2040 |
|
| 2041 | VM_WARN_ON_ONCE(!(flags & TTU_RMAP_LOCKED));
|
| 2042 | if (!hugetlb_vma_trylock_write(vma)) {
|
| 2043 | ret = false;
|
| 2044 | goto walk_done;
|
| 2045 | }
|
| 2046 |
|
| 2047 | tlb_gather_mmu_vma(&tlb, vma);
|
| 2048 | if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
|
| 2049 | hugetlb_vma_unlock_write(vma);
|
| 2050 | huge_pmd_unshare_flush(&tlb, vma);
|
| 2051 | tlb_finish_mmu(&tlb);
|
| 2052 | |
| 2053 | * The PMD table was unmapped, consequently unmapping
|
| 2054 | * the folio.
|
| 2055 |
|
| 2056 | goto walk_done;
|
| 2057 | }
|
| 2058 | hugetlb_vma_unlock_write(vma);
|
| 2059 | tlb_finish_mmu(&tlb);
|
| 2060 | }
|
| 2061 | pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
|
| 2062 | if (huge_pte_dirty(pteval))
|
| 2063 | folio_mark_dirty(folio);
|
| 2064 |
|
| 2065 | pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0)));
|
| 2066 | hugetlb_count_sub(folio_nr_pages(folio), mm);
|
| 2067 | set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz);
|
| 2068 | hugetlb_remove_rmap(folio);
|
| 2069 | folio_put_refs(folio, 1);
|
| 2070 |
|
| 2071 | walk_done:
|
| 2072 | page_vma_mapped_walk_done(&pvmw);
|
| 2073 | range_end:
|
| 2074 | mmu_notifier_invalidate_range_end(&range);
|
| 2075 | return ret;
|
| 2076 | }
|
| 2077 |
|
| 2078 | static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma,
|
| 2079 | struct folio *folio, unsigned long nr_pages)
|
| 2080 | {
|
| 2081 | int ref_count, map_count;
|
| 2082 |
|
| 2083 | |
| 2084 | * Synchronize with gup_pte_range():
|
| 2085 | * - clear PTE; barrier; read refcount
|
| 2086 | * - inc refcount; barrier; read PTE
|
| 2087 |
|
| 2088 | smp_mb();
|
| 2089 |
|
| 2090 | ref_count = folio_ref_count(folio);
|
| 2091 | map_count = folio_mapcount(folio);
|
| 2092 |
|
| 2093 | |
| 2094 | * Order reads for page refcount and dirty flag
|
| 2095 | * (see comments in __remove_mapping()).
|
| 2096 |
|
| 2097 | smp_rmb();
|
| 2098 |
|
| 2099 | if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
|
| 2100 | |
| 2101 | * redirtied either using the page table or a previously
|
| 2102 | * obtained GUP reference.
|
| 2103 |
|
| 2104 | folio_set_swapbacked(folio);
|
| 2105 | return false;
|
| 2106 | }
|
| 2107 |
|
| 2108 | |
| 2109 | * Additional references could be due to GUP or speculative lookups.
|
| 2110 | * GUP users must mark the folio dirty if there was a modification.
|
| 2111 | * This folio cannot be reclaimed right now either way, so act just
|
| 2112 | * like nothing happened. We'll come back here later and detect if the
|
| 2113 | * folio was dirtied when the additional reference is gone.
|
| 2114 |
|
| 2115 | if (ref_count != 1 + map_count)
|
| 2116 | return false;
|
| 2117 |
|
| 2118 | add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages);
|
| 2119 | return true;
|
| 2120 | }
|
| 2121 |
|
| 2122 | static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte,
|
| 2123 | bool anon_exclusive)
|
| 2124 | {
|
| 2125 | pte_t swp_pte = swp_entry_to_pte(entry);
|
| 2126 |
|
| 2127 | if (anon_exclusive)
|
| 2128 | swp_pte = pte_swp_mkexclusive(swp_pte);
|
| 2129 |
|
| 2130 | if (likely(pte_present(old_pte))) {
|
| 2131 | if (pte_soft_dirty(old_pte))
|
| 2132 | swp_pte = pte_swp_mksoft_dirty(swp_pte);
|
| 2133 | if (pte_uffd(old_pte))
|
| 2134 | swp_pte = pte_swp_mkuffd(swp_pte);
|
| 2135 | } else {
|
| 2136 |
|
| 2137 | if (pte_swp_soft_dirty(old_pte))
|
| 2138 | swp_pte = pte_swp_mksoft_dirty(swp_pte);
|
| 2139 | if (pte_swp_uffd(old_pte))
|
| 2140 | swp_pte = pte_swp_mkuffd(swp_pte);
|
| 2141 | }
|
| 2142 |
|
| 2143 | return swp_pte;
|
| 2144 | }
|
| 2145 |
|
| 2146 | static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
|
| 2147 | struct folio *folio, struct page *page, unsigned long address,
|
| 2148 | pte_t *ptep, pte_t pteval)
|
| 2149 | {
|
| 2150 | const bool anon_exclusive = folio_test_anon(folio) &&
|
| 2151 | PageAnonExclusive(page);
|
| 2152 | swp_entry_t entry = page_swap_entry(page);
|
| 2153 | struct mm_struct *mm = vma->vm_mm;
|
| 2154 |
|
| 2155 | if (folio_dup_swap(folio, page) < 0)
|
| 2156 | return false;
|
| 2157 |
|
| 2158 | |
| 2159 | * arch_unmap_one() is expected to be a NOP on
|
| 2160 | * architectures where we could have PFN swap PTEs,
|
| 2161 | * so we'll not check/care.
|
| 2162 |
|
| 2163 | if (arch_unmap_one(mm, vma, address, pteval) < 0) {
|
| 2164 | folio_put_swap(folio, page);
|
| 2165 | return false;
|
| 2166 | }
|
| 2167 |
|
| 2168 |
|
| 2169 | if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) {
|
| 2170 | folio_put_swap(folio, page);
|
| 2171 | return false;
|
| 2172 | }
|
| 2173 |
|
| 2174 | mm_prepare_for_swap_entries(mm);
|
| 2175 | dec_mm_counter(mm, MM_ANONPAGES);
|
| 2176 | inc_mm_counter(mm, MM_SWAPENTS);
|
| 2177 | set_pte_at(mm, address, ptep,
|
| 2178 | swp_pte_prepare(entry, pteval, anon_exclusive));
|
| 2179 | return true;
|
| 2180 | }
|
| 2181 |
|
| 2182 | static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,
|
| 2183 | struct page *page, unsigned long address, pte_t *ptep,
|
| 2184 | pte_t pteval, unsigned long nr_pages)
|
| 2185 | {
|
| 2186 | |
| 2187 | * Store the swap location in the pte.
|
| 2188 | * See handle_pte_fault() ...
|
| 2189 |
|
| 2190 | if (WARN_ON_ONCE(folio_test_swapbacked(folio) !=
|
| 2191 | folio_test_swapcache(folio)))
|
| 2192 | return false;
|
| 2193 |
|
| 2194 | if (!folio_test_swapbacked(folio))
|
| 2195 | return ttu_anon_lazyfree_folio(vma, folio, nr_pages);
|
| 2196 |
|
| 2197 |
|
| 2198 | return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep,
|
| 2199 | pteval);
|
| 2200 | }
|
| 2201 |
|
| 2202 | |
| 2203 | * @arg: enum ttu_flags will be passed to this argument
|
| 2204 |
|
| 2205 | static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
|
| 2206 | unsigned long address, void *arg)
|
| 2207 | {
|
| 2208 | struct mm_struct *mm = vma->vm_mm;
|
| 2209 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
|
| 2210 | bool ret = true;
|
| 2211 | pte_t pteval;
|
| 2212 | struct page *page;
|
| 2213 | struct mmu_notifier_range range;
|
| 2214 | enum ttu_flags flags = (enum ttu_flags)(long)arg;
|
| 2215 | unsigned long nr_pages = 1, end_addr;
|
| 2216 | unsigned long pfn;
|
| 2217 | int ptes = 0;
|
| 2218 |
|
| 2219 | |
| 2220 | * When racing against e.g. zap_pte_range() on another cpu,
|
| 2221 | * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
|
| 2222 | * try_to_unmap() may return before folio_mapped() has become false,
|
| 2223 | * if page table locking is skipped: use TTU_SYNC to wait for that.
|
| 2224 |
|
| 2225 | if (flags & TTU_SYNC)
|
| 2226 | pvmw.flags = PVMW_SYNC;
|
| 2227 |
|
| 2228 | |
| 2229 | * For THP, we have to assume the worse case ie pmd for invalidation.
|
| 2230 | *
|
| 2231 | * Note that the folio can not be freed in this function as call of
|
| 2232 | * try_to_unmap() must hold a reference on the folio.
|
| 2233 |
|
| 2234 | range.end = vma_address_end(&pvmw);
|
| 2235 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
| 2236 | address, range.end);
|
| 2237 | mmu_notifier_invalidate_range_start(&range);
|
| 2238 |
|
| 2239 | while (page_vma_mapped_walk(&pvmw)) {
|
| 2240 | nr_pages = 1;
|
| 2241 |
|
| 2242 | |
| 2243 | * If the folio is in an mlock()d vma, we must not swap it out.
|
| 2244 |
|
| 2245 | if (!(flags & TTU_IGNORE_MLOCK) &&
|
| 2246 | (vma->vm_flags & VM_LOCKED)) {
|
| 2247 | ptes++;
|
| 2248 |
|
| 2249 | |
| 2250 | * Set 'ret' to indicate the page cannot be unmapped.
|
| 2251 | *
|
| 2252 | * Do not jump to walk_abort immediately as additional
|
| 2253 | * iteration might be required to detect fully mapped
|
| 2254 | * folio an mlock it.
|
| 2255 |
|
| 2256 | ret = false;
|
| 2257 |
|
| 2258 |
|
| 2259 | if (pvmw.pte && ptes != pvmw.nr_pages)
|
| 2260 | continue;
|
| 2261 |
|
| 2262 | |
| 2263 | * All PTEs must be protected by page table lock in
|
| 2264 | * order to mlock the page.
|
| 2265 | *
|
| 2266 | * If page table boundary has been cross, current ptl
|
| 2267 | * only protect part of ptes.
|
| 2268 |
|
| 2269 | if (pvmw.flags & PVMW_PGTABLE_CROSSED)
|
| 2270 | goto walk_done;
|
| 2271 |
|
| 2272 |
|
| 2273 | mlock_vma_folio(folio, vma);
|
| 2274 | goto walk_done;
|
| 2275 | }
|
| 2276 |
|
| 2277 | if (!pvmw.pte) {
|
| 2278 | if (folio_test_lazyfree(folio)) {
|
| 2279 | if (unmap_huge_pmd_locked(vma, pvmw.address, pvmw.pmd, folio))
|
| 2280 | goto walk_done;
|
| 2281 | |
| 2282 | * unmap_huge_pmd_locked has either already marked
|
| 2283 | * the folio as swap-backed or decided to retain it
|
| 2284 | * due to GUP or speculative references.
|
| 2285 |
|
| 2286 | goto walk_abort;
|
| 2287 | }
|
| 2288 |
|
| 2289 | if (flags & TTU_SPLIT_HUGE_PMD) {
|
| 2290 | |
| 2291 | * We temporarily have to drop the PTL and
|
| 2292 | * restart so we can process the PTE-mapped THP.
|
| 2293 |
|
| 2294 | split_huge_pmd_locked(vma, pvmw.address,
|
| 2295 | pvmw.pmd, false);
|
| 2296 | flags &= ~TTU_SPLIT_HUGE_PMD;
|
| 2297 | page_vma_mapped_walk_restart(&pvmw);
|
| 2298 | continue;
|
| 2299 | }
|
| 2300 | }
|
| 2301 |
|
| 2302 |
|
| 2303 | VM_BUG_ON_FOLIO(!pvmw.pte, folio);
|
| 2304 |
|
| 2305 | address = pvmw.address;
|
| 2306 | if (folio_test_hugetlb(folio)) {
|
| 2307 | pteval = huge_ptep_get(mm, address, pvmw.pte);
|
| 2308 | } else {
|
| 2309 | pteval = ptep_get(pvmw.pte);
|
| 2310 | }
|
| 2311 | if (likely(pte_present(pteval))) {
|
| 2312 | pfn = pte_pfn(pteval);
|
| 2313 | } else {
|
| 2314 | |
| 2315 | * Handle PFN swap PTEs, such as device-exclusive ones,
|
| 2316 | * that actually map pages.
|
| 2317 |
|
| 2318 | const softleaf_t entry = softleaf_from_pte(pteval);
|
| 2319 |
|
| 2320 | pfn = softleaf_to_pfn(entry);
|
| 2321 | }
|
| 2322 |
|
| 2323 | page = folio_page(folio, pfn - folio_pfn(folio));
|
| 2324 |
|
| 2325 | if (likely(pte_present(pteval))) {
|
| 2326 | nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval);
|
| 2327 | end_addr = address + nr_pages * PAGE_SIZE;
|
| 2328 | flush_cache_range(vma, address, end_addr);
|
| 2329 |
|
| 2330 |
|
| 2331 | pteval = get_and_clear_ptes(mm, address, pvmw.pte, nr_pages);
|
| 2332 | |
| 2333 | * We clear the PTE but do not flush so potentially
|
| 2334 | * a remote CPU could still be writing to the folio.
|
| 2335 | * If the entry was previously clean then the
|
| 2336 | * architecture must guarantee that a clear->dirty
|
| 2337 | * transition on a cached TLB entry is written through
|
| 2338 | * and traps if the PTE is unmapped.
|
| 2339 |
|
| 2340 | if (should_defer_flush(mm, flags))
|
| 2341 | set_tlb_ubc_flush_pending(mm, pteval, address, end_addr);
|
| 2342 | else
|
| 2343 | flush_tlb_range(vma, address, end_addr);
|
| 2344 | if (pte_dirty(pteval))
|
| 2345 | folio_mark_dirty(folio);
|
| 2346 | } else {
|
| 2347 | pte_clear(mm, address, pvmw.pte);
|
| 2348 | }
|
| 2349 |
|
| 2350 | |
| 2351 | * Now the pte is cleared. If this pte was uffd-wp armed,
|
| 2352 | * we may want to replace a none pte with a marker pte if
|
| 2353 | * it's file-backed, so we don't lose the tracking info.
|
| 2354 |
|
| 2355 | cond_install_uffd_wp_ptes(vma, address, pvmw.pte, pteval,
|
| 2356 | nr_pages);
|
| 2357 |
|
| 2358 |
|
| 2359 | update_hiwater_rss(mm);
|
| 2360 |
|
| 2361 |
|
| 2362 | if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) {
|
| 2363 | pteval = swp_entry_to_pte(make_hwpoison_entry(page));
|
| 2364 | dec_mm_counter(mm, mm_counter(folio));
|
| 2365 | set_pte_at(mm, address, pvmw.pte, pteval);
|
| 2366 | } else if (likely(pte_present(pteval)) && pte_unused(pteval) &&
|
| 2367 | !userfaultfd_armed(vma)) {
|
| 2368 | |
| 2369 | * The guest indicated that the page content is of no
|
| 2370 | * interest anymore. Simply discard the pte, vmscan
|
| 2371 | * will take care of the rest.
|
| 2372 | * A future reference will then fault in a new zero
|
| 2373 | * page. When userfaultfd is active, we must not drop
|
| 2374 | * this page though, as its main user (postcopy
|
| 2375 | * migration) will not expect userfaults on already
|
| 2376 | * copied pages.
|
| 2377 |
|
| 2378 | dec_mm_counter(mm, mm_counter(folio));
|
| 2379 | } else if (folio_test_anon(folio)) {
|
| 2380 | if (!ttu_anon_folio(vma, folio, page, address,
|
| 2381 | pvmw.pte, pteval, nr_pages)) {
|
| 2382 | set_ptes(mm, address, pvmw.pte, pteval, nr_pages);
|
| 2383 | goto walk_abort;
|
| 2384 | }
|
| 2385 |
|
| 2386 | goto finish_unmap;
|
| 2387 | } else {
|
| 2388 | |
| 2389 | * This is a locked file-backed folio,
|
| 2390 | * so it cannot be removed from the page
|
| 2391 | * cache and replaced by a new folio before
|
| 2392 | * mmu_notifier_invalidate_range_end, so no
|
| 2393 | * concurrent thread might update its page table
|
| 2394 | * to point at a new folio while a device is
|
| 2395 | * still using this folio.
|
| 2396 | *
|
| 2397 | * See Documentation/mm/mmu_notifier.rst
|
| 2398 |
|
| 2399 | add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
|
| 2400 | }
|
| 2401 | finish_unmap:
|
| 2402 | folio_remove_rmap_ptes(folio, page, nr_pages, vma);
|
| 2403 | if (vma->vm_flags & VM_LOCKED)
|
| 2404 | mlock_drain_local();
|
| 2405 | folio_put_refs(folio, nr_pages);
|
| 2406 |
|
| 2407 | |
| 2408 | * If we are sure that we batched the entire folio and cleared
|
| 2409 | * all PTEs, we can just optimize and stop right here.
|
| 2410 |
|
| 2411 | if (nr_pages == folio_nr_pages(folio))
|
| 2412 | goto walk_done;
|
| 2413 | continue;
|
| 2414 | walk_abort:
|
| 2415 | ret = false;
|
| 2416 | walk_done:
|
| 2417 | page_vma_mapped_walk_done(&pvmw);
|
| 2418 | break;
|
| 2419 | }
|
| 2420 |
|
| 2421 | mmu_notifier_invalidate_range_end(&range);
|
| 2422 |
|
| 2423 | return ret;
|
| 2424 | }
|
| 2425 |
|
| 2426 | static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
|
| 2427 | {
|
| 2428 | return vma_is_temporary_stack(vma);
|
| 2429 | }
|
| 2430 |
|
| 2431 | static int folio_not_mapped(struct folio *folio)
|
| 2432 | {
|
| 2433 | return !folio_mapped(folio);
|
| 2434 | }
|
| 2435 |
|
| 2436 | |
| 2437 | * try_to_unmap - Try to remove all page table mappings to a folio.
|
| 2438 | * @folio: The folio to unmap.
|
| 2439 | * @flags: action and flags
|
| 2440 | *
|
| 2441 | * Tries to remove all the page table entries which are mapping this
|
| 2442 | * folio. It is the caller's responsibility to check if the folio is
|
| 2443 | * still mapped if needed (use TTU_SYNC to prevent accounting races).
|
| 2444 | *
|
| 2445 | * Context: Caller must hold the folio lock.
|
| 2446 |
|
| 2447 | void try_to_unmap(struct folio *folio, enum ttu_flags flags)
|
| 2448 | {
|
| 2449 | struct rmap_walk_control rwc = {
|
| 2450 | .rmap_one = folio_test_hugetlb(folio) ?
|
| 2451 | try_to_unmap_poisoned_hugetlb_one : try_to_unmap_one,
|
| 2452 | .arg = (void *)flags,
|
| 2453 | .done = folio_not_mapped,
|
| 2454 | .anon_lock = folio_lock_anon_vma_read,
|
| 2455 | };
|
| 2456 |
|
| 2457 | if (flags & TTU_RMAP_LOCKED)
|
| 2458 | rmap_walk_locked(folio, &rwc);
|
| 2459 | else
|
| 2460 | rmap_walk(folio, &rwc);
|
| 2461 | }
|
| 2462 |
|
| 2463 | |
| 2464 | * @arg: enum ttu_flags will be passed to this argument.
|
| 2465 | *
|
| 2466 | * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs
|
| 2467 | * containing migration entries.
|
| 2468 |
|
| 2469 | static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
|
| 2470 | unsigned long address, void *arg)
|
| 2471 | {
|
| 2472 | struct mm_struct *mm = vma->vm_mm;
|
| 2473 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
|
| 2474 | bool anon_exclusive, writable, ret = true;
|
| 2475 | pte_t pteval;
|
| 2476 | struct page *subpage;
|
| 2477 | struct mmu_notifier_range range;
|
| 2478 | enum ttu_flags flags = (enum ttu_flags)(long)arg;
|
| 2479 | unsigned long pfn;
|
| 2480 | unsigned long hsz = 0;
|
| 2481 |
|
| 2482 | |
| 2483 | * When racing against e.g. zap_pte_range() on another cpu,
|
| 2484 | * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
|
| 2485 | * try_to_migrate() may return before folio_mapped() has become false,
|
| 2486 | * if page table locking is skipped: use TTU_SYNC to wait for that.
|
| 2487 |
|
| 2488 | if (flags & TTU_SYNC)
|
| 2489 | pvmw.flags = PVMW_SYNC;
|
| 2490 |
|
| 2491 | |
| 2492 | * For THP, we have to assume the worse case ie pmd for invalidation.
|
| 2493 | * For hugetlb, it could be much worse if we need to do pud
|
| 2494 | * invalidation in the case of pmd sharing.
|
| 2495 | *
|
| 2496 | * Note that the page can not be free in this function as call of
|
| 2497 | * try_to_unmap() must hold a reference on the page.
|
| 2498 |
|
| 2499 | range.end = vma_address_end(&pvmw);
|
| 2500 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
| 2501 | address, range.end);
|
| 2502 | if (folio_test_hugetlb(folio)) {
|
| 2503 | |
| 2504 | * If sharing is possible, start and end will be adjusted
|
| 2505 | * accordingly.
|
| 2506 |
|
| 2507 | adjust_range_if_pmd_sharing_possible(vma, &range.start,
|
| 2508 | &range.end);
|
| 2509 |
|
| 2510 |
|
| 2511 | hsz = huge_page_size(hstate_vma(vma));
|
| 2512 | }
|
| 2513 | mmu_notifier_invalidate_range_start(&range);
|
| 2514 |
|
| 2515 | while (page_vma_mapped_walk(&pvmw)) {
|
| 2516 |
|
| 2517 | if (!pvmw.pte) {
|
| 2518 | __maybe_unused unsigned long pfn;
|
| 2519 | __maybe_unused pmd_t pmdval;
|
| 2520 |
|
| 2521 | if (flags & TTU_SPLIT_HUGE_PMD) {
|
| 2522 | |
| 2523 | * split_huge_pmd_locked() might leave the
|
| 2524 | * folio mapped through PTEs. Retry the walk
|
| 2525 | * so we can detect this scenario and properly
|
| 2526 | * abort the walk.
|
| 2527 |
|
| 2528 | split_huge_pmd_locked(vma, pvmw.address,
|
| 2529 | pvmw.pmd, true);
|
| 2530 | flags &= ~TTU_SPLIT_HUGE_PMD;
|
| 2531 | page_vma_mapped_walk_restart(&pvmw);
|
| 2532 | continue;
|
| 2533 | }
|
| 2534 | #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
|
| 2535 | pmdval = pmdp_get(pvmw.pmd);
|
| 2536 | if (likely(pmd_present(pmdval)))
|
| 2537 | pfn = pmd_pfn(pmdval);
|
| 2538 | else
|
| 2539 | pfn = softleaf_to_pfn(softleaf_from_pmd(pmdval));
|
| 2540 |
|
| 2541 | subpage = folio_page(folio, pfn - folio_pfn(folio));
|
| 2542 |
|
| 2543 | VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
|
| 2544 | !folio_test_pmd_mappable(folio), folio);
|
| 2545 |
|
| 2546 | if (set_pmd_migration_entry(&pvmw, subpage)) {
|
| 2547 | ret = false;
|
| 2548 | page_vma_mapped_walk_done(&pvmw);
|
| 2549 | break;
|
| 2550 | }
|
| 2551 | continue;
|
| 2552 | #endif
|
| 2553 | }
|
| 2554 |
|
| 2555 |
|
| 2556 | VM_BUG_ON_FOLIO(!pvmw.pte, folio);
|
| 2557 |
|
| 2558 | address = pvmw.address;
|
| 2559 | if (folio_test_hugetlb(folio))
|
| 2560 | pteval = huge_ptep_get(mm, address, pvmw.pte);
|
| 2561 | else
|
| 2562 | pteval = ptep_get(pvmw.pte);
|
| 2563 | if (likely(pte_present(pteval))) {
|
| 2564 | pfn = pte_pfn(pteval);
|
| 2565 | } else {
|
| 2566 | |
| 2567 | * Handle PFN swap PTEs, such as device-exclusive ones,
|
| 2568 | * that actually map pages.
|
| 2569 |
|
| 2570 | const softleaf_t entry = softleaf_from_pte(pteval);
|
| 2571 |
|
| 2572 | pfn = softleaf_to_pfn(entry);
|
| 2573 | VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
|
| 2574 | }
|
| 2575 |
|
| 2576 | subpage = folio_page(folio, pfn - folio_pfn(folio));
|
| 2577 | anon_exclusive = folio_test_anon(folio) &&
|
| 2578 | PageAnonExclusive(subpage);
|
| 2579 |
|
| 2580 | if (folio_test_hugetlb(folio)) {
|
| 2581 | bool anon = folio_test_anon(folio);
|
| 2582 |
|
| 2583 | |
| 2584 | * huge_pmd_unshare may unmap an entire PMD page.
|
| 2585 | * There is no way of knowing exactly which PMDs may
|
| 2586 | * be cached for this mm, so we must flush them all.
|
| 2587 | * start/end were already adjusted above to cover this
|
| 2588 | * range.
|
| 2589 |
|
| 2590 | flush_cache_range(vma, range.start, range.end);
|
| 2591 |
|
| 2592 | |
| 2593 | * To call huge_pmd_unshare, i_mmap_rwsem must be
|
| 2594 | * held in write mode. Caller needs to explicitly
|
| 2595 | * do this outside rmap routines.
|
| 2596 | *
|
| 2597 | * We also must hold hugetlb vma_lock in write mode.
|
| 2598 | * Lock order dictates acquiring vma_lock BEFORE
|
| 2599 | * i_mmap_rwsem. We can only try lock here and
|
| 2600 | * fail if unsuccessful.
|
| 2601 |
|
| 2602 | if (!anon) {
|
| 2603 | struct mmu_gather tlb;
|
| 2604 |
|
| 2605 | VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
|
| 2606 | if (!hugetlb_vma_trylock_write(vma)) {
|
| 2607 | page_vma_mapped_walk_done(&pvmw);
|
| 2608 | ret = false;
|
| 2609 | break;
|
| 2610 | }
|
| 2611 |
|
| 2612 | tlb_gather_mmu_vma(&tlb, vma);
|
| 2613 | if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
|
| 2614 | hugetlb_vma_unlock_write(vma);
|
| 2615 | huge_pmd_unshare_flush(&tlb, vma);
|
| 2616 | tlb_finish_mmu(&tlb);
|
| 2617 | |
| 2618 | * The PMD table was unmapped,
|
| 2619 | * consequently unmapping the folio.
|
| 2620 |
|
| 2621 | page_vma_mapped_walk_done(&pvmw);
|
| 2622 | break;
|
| 2623 | }
|
| 2624 | hugetlb_vma_unlock_write(vma);
|
| 2625 | tlb_finish_mmu(&tlb);
|
| 2626 | }
|
| 2627 |
|
| 2628 | pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
|
| 2629 | if (pte_dirty(pteval))
|
| 2630 | folio_mark_dirty(folio);
|
| 2631 | writable = pte_write(pteval);
|
| 2632 | } else if (likely(pte_present(pteval))) {
|
| 2633 | flush_cache_page(vma, address, pfn);
|
| 2634 |
|
| 2635 | if (should_defer_flush(mm, flags)) {
|
| 2636 | |
| 2637 | * We clear the PTE but do not flush so potentially
|
| 2638 | * a remote CPU could still be writing to the folio.
|
| 2639 | * If the entry was previously clean then the
|
| 2640 | * architecture must guarantee that a clear->dirty
|
| 2641 | * transition on a cached TLB entry is written through
|
| 2642 | * and traps if the PTE is unmapped.
|
| 2643 |
|
| 2644 | pteval = ptep_get_and_clear(mm, address, pvmw.pte);
|
| 2645 |
|
| 2646 | set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE);
|
| 2647 | } else {
|
| 2648 | pteval = ptep_clear_flush(vma, address, pvmw.pte);
|
| 2649 | }
|
| 2650 | if (pte_dirty(pteval))
|
| 2651 | folio_mark_dirty(folio);
|
| 2652 | writable = pte_write(pteval);
|
| 2653 | } else {
|
| 2654 | const softleaf_t entry = softleaf_from_pte(pteval);
|
| 2655 |
|
| 2656 | pte_clear(mm, address, pvmw.pte);
|
| 2657 |
|
| 2658 | writable = softleaf_is_device_private_write(entry);
|
| 2659 | }
|
| 2660 |
|
| 2661 | VM_WARN_ON_FOLIO(writable && folio_test_anon(folio) &&
|
| 2662 | !anon_exclusive, folio);
|
| 2663 |
|
| 2664 |
|
| 2665 | update_hiwater_rss(mm);
|
| 2666 |
|
| 2667 | if (PageHWPoison(subpage)) {
|
| 2668 | VM_WARN_ON_FOLIO(folio_is_device_private(folio), folio);
|
| 2669 |
|
| 2670 | pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
|
| 2671 | if (folio_test_hugetlb(folio)) {
|
| 2672 | hugetlb_count_sub(folio_nr_pages(folio), mm);
|
| 2673 | set_huge_pte_at(mm, address, pvmw.pte, pteval,
|
| 2674 | hsz);
|
| 2675 | } else {
|
| 2676 | dec_mm_counter(mm, mm_counter(folio));
|
| 2677 | set_pte_at(mm, address, pvmw.pte, pteval);
|
| 2678 | }
|
| 2679 | } else if (likely(pte_present(pteval)) && pte_unused(pteval) &&
|
| 2680 | !userfaultfd_armed(vma)) {
|
| 2681 | |
| 2682 | * The guest indicated that the page content is of no
|
| 2683 | * interest anymore. Simply discard the pte, vmscan
|
| 2684 | * will take care of the rest.
|
| 2685 | * A future reference will then fault in a new zero
|
| 2686 | * page. When userfaultfd is active, we must not drop
|
| 2687 | * this page though, as its main user (postcopy
|
| 2688 | * migration) will not expect userfaults on already
|
| 2689 | * copied pages.
|
| 2690 |
|
| 2691 | dec_mm_counter(mm, mm_counter(folio));
|
| 2692 | } else {
|
| 2693 | swp_entry_t entry;
|
| 2694 | pte_t swp_pte;
|
| 2695 |
|
| 2696 | |
| 2697 | * arch_unmap_one() is expected to be a NOP on
|
| 2698 | * architectures where we could have PFN swap PTEs,
|
| 2699 | * so we'll not check/care.
|
| 2700 |
|
| 2701 | if (arch_unmap_one(mm, vma, address, pteval) < 0) {
|
| 2702 | if (folio_test_hugetlb(folio))
|
| 2703 | set_huge_pte_at(mm, address, pvmw.pte,
|
| 2704 | pteval, hsz);
|
| 2705 | else
|
| 2706 | set_pte_at(mm, address, pvmw.pte, pteval);
|
| 2707 | ret = false;
|
| 2708 | page_vma_mapped_walk_done(&pvmw);
|
| 2709 | break;
|
| 2710 | }
|
| 2711 |
|
| 2712 |
|
| 2713 | if (folio_test_hugetlb(folio)) {
|
| 2714 | if (anon_exclusive &&
|
| 2715 | hugetlb_try_share_anon_rmap(folio)) {
|
| 2716 | set_huge_pte_at(mm, address, pvmw.pte,
|
| 2717 | pteval, hsz);
|
| 2718 | ret = false;
|
| 2719 | page_vma_mapped_walk_done(&pvmw);
|
| 2720 | break;
|
| 2721 | }
|
| 2722 | } else if (anon_exclusive &&
|
| 2723 | folio_try_share_anon_rmap_pte(folio, subpage)) {
|
| 2724 | set_pte_at(mm, address, pvmw.pte, pteval);
|
| 2725 | ret = false;
|
| 2726 | page_vma_mapped_walk_done(&pvmw);
|
| 2727 | break;
|
| 2728 | }
|
| 2729 |
|
| 2730 | |
| 2731 | * Store the pfn of the page in a special migration
|
| 2732 | * pte. do_swap_page() will wait until the migration
|
| 2733 | * pte is removed and then restart fault handling.
|
| 2734 |
|
| 2735 | if (writable)
|
| 2736 | entry = make_writable_migration_entry(
|
| 2737 | page_to_pfn(subpage));
|
| 2738 | else if (anon_exclusive)
|
| 2739 | entry = make_readable_exclusive_migration_entry(
|
| 2740 | page_to_pfn(subpage));
|
| 2741 | else
|
| 2742 | entry = make_readable_migration_entry(
|
| 2743 | page_to_pfn(subpage));
|
| 2744 | if (likely(pte_present(pteval))) {
|
| 2745 | if (pte_young(pteval))
|
| 2746 | entry = make_migration_entry_young(entry);
|
| 2747 | if (pte_dirty(pteval))
|
| 2748 | entry = make_migration_entry_dirty(entry);
|
| 2749 | swp_pte = swp_entry_to_pte(entry);
|
| 2750 | if (pte_soft_dirty(pteval))
|
| 2751 | swp_pte = pte_swp_mksoft_dirty(swp_pte);
|
| 2752 | if (pte_uffd(pteval))
|
| 2753 | swp_pte = pte_swp_mkuffd(swp_pte);
|
| 2754 | } else {
|
| 2755 | swp_pte = swp_entry_to_pte(entry);
|
| 2756 | if (pte_swp_soft_dirty(pteval))
|
| 2757 | swp_pte = pte_swp_mksoft_dirty(swp_pte);
|
| 2758 | if (pte_swp_uffd(pteval))
|
| 2759 | swp_pte = pte_swp_mkuffd(swp_pte);
|
| 2760 | }
|
| 2761 | if (folio_test_hugetlb(folio))
|
| 2762 | set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
|
| 2763 | hsz);
|
| 2764 | else
|
| 2765 | set_pte_at(mm, address, pvmw.pte, swp_pte);
|
| 2766 | trace_set_migration_pte(address, pte_val(swp_pte),
|
| 2767 | folio_order(folio));
|
| 2768 | |
| 2769 | * No need to invalidate here it will synchronize on
|
| 2770 | * against the special swap migration pte.
|
| 2771 |
|
| 2772 | }
|
| 2773 |
|
| 2774 | if (unlikely(folio_test_hugetlb(folio)))
|
| 2775 | hugetlb_remove_rmap(folio);
|
| 2776 | else
|
| 2777 | folio_remove_rmap_pte(folio, subpage, vma);
|
| 2778 | if (vma->vm_flags & VM_LOCKED)
|
| 2779 | mlock_drain_local();
|
| 2780 | folio_put(folio);
|
| 2781 | }
|
| 2782 |
|
| 2783 | mmu_notifier_invalidate_range_end(&range);
|
| 2784 |
|
| 2785 | return ret;
|
| 2786 | }
|
| 2787 |
|
| 2788 | |
| 2789 | * try_to_migrate - try to replace all page table mappings with swap entries
|
| 2790 | * @folio: the folio to replace page table entries for
|
| 2791 | * @flags: action and flags
|
| 2792 | *
|
| 2793 | * Tries to remove all the page table entries which are mapping this folio and
|
| 2794 | * replace them with special swap entries. Caller must hold the folio lock.
|
| 2795 |
|
| 2796 | void try_to_migrate(struct folio *folio, enum ttu_flags flags)
|
| 2797 | {
|
| 2798 | struct rmap_walk_control rwc = {
|
| 2799 | .rmap_one = try_to_migrate_one,
|
| 2800 | .arg = (void *)flags,
|
| 2801 | .done = folio_not_mapped,
|
| 2802 | .anon_lock = folio_lock_anon_vma_read,
|
| 2803 | };
|
| 2804 |
|
| 2805 | |
| 2806 | * Migration always ignores mlock and only supports TTU_RMAP_LOCKED and
|
| 2807 | * TTU_SPLIT_HUGE_PMD, TTU_SYNC, and TTU_BATCH_FLUSH flags.
|
| 2808 |
|
| 2809 | if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
|
| 2810 | TTU_SYNC | TTU_BATCH_FLUSH)))
|
| 2811 | return;
|
| 2812 |
|
| 2813 | if (folio_is_zone_device(folio) &&
|
| 2814 | (!folio_is_device_private(folio) && !folio_is_device_coherent(folio)))
|
| 2815 | return;
|
| 2816 |
|
| 2817 | |
| 2818 | * During exec, a temporary VMA is setup and later moved.
|
| 2819 | * The VMA is moved under the anon_vma lock but not the
|
| 2820 | * page tables leading to a race where migration cannot
|
| 2821 | * find the migration ptes. Rather than increasing the
|
| 2822 | * locking requirements of exec(), migration skips
|
| 2823 | * temporary VMAs until after exec() completes.
|
| 2824 |
|
| 2825 | if (!folio_test_ksm(folio) && folio_test_anon(folio))
|
| 2826 | rwc.invalid_vma = invalid_migration_vma;
|
| 2827 |
|
| 2828 | if (flags & TTU_RMAP_LOCKED)
|
| 2829 | rmap_walk_locked(folio, &rwc);
|
| 2830 | else
|
| 2831 | rmap_walk(folio, &rwc);
|
| 2832 | }
|
| 2833 |
|
| 2834 | #ifdef CONFIG_DEVICE_PRIVATE
|
| 2835 | |
| 2836 | * make_device_exclusive() - Mark a page for exclusive use by a device
|
| 2837 | * @mm: mm_struct of associated target process
|
| 2838 | * @addr: the virtual address to mark for exclusive device access
|
| 2839 | * @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier to allow filtering
|
| 2840 | * @foliop: folio pointer will be stored here on success.
|
| 2841 | *
|
| 2842 | * This function looks up the page mapped at the given address, grabs a
|
| 2843 | * folio reference, locks the folio and replaces the PTE with special
|
| 2844 | * device-exclusive PFN swap entry, preventing access through the process
|
| 2845 | * page tables. The function will return with the folio locked and referenced.
|
| 2846 | *
|
| 2847 | * On fault, the device-exclusive entries are replaced with the original PTE
|
| 2848 | * under folio lock, after calling MMU notifiers.
|
| 2849 | *
|
| 2850 | * Only anonymous non-hugetlb folios are supported and the VMA must have
|
| 2851 | * write permissions such that we can fault in the anonymous page writable
|
| 2852 | * in order to mark it exclusive. The caller must hold the mmap_lock in read
|
| 2853 | * mode.
|
| 2854 | *
|
| 2855 | * A driver using this to program access from a device must use a mmu notifier
|
| 2856 | * critical section to hold a device specific lock during programming. Once
|
| 2857 | * programming is complete it should drop the folio lock and reference after
|
| 2858 | * which point CPU access to the page will revoke the exclusive access.
|
| 2859 | *
|
| 2860 | * Notes:
|
| 2861 | * #. This function always operates on individual PTEs mapping individual
|
| 2862 | * pages. PMD-sized THPs are first remapped to be mapped by PTEs before
|
| 2863 | * the conversion happens on a single PTE corresponding to @addr.
|
| 2864 | * #. While concurrent access through the process page tables is prevented,
|
| 2865 | * concurrent access through other page references (e.g., earlier GUP
|
| 2866 | * invocation) is not handled and not supported.
|
| 2867 | * #. device-exclusive entries are considered "clean" and "old" by core-mm.
|
| 2868 | * Device drivers must update the folio state when informed by MMU
|
| 2869 | * notifiers.
|
| 2870 | *
|
| 2871 | * Returns: pointer to mapped page on success, otherwise a negative error.
|
| 2872 |
|
| 2873 | struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr,
|
| 2874 | void *owner, struct folio **foliop)
|
| 2875 | {
|
| 2876 | struct mmu_notifier_range range;
|
| 2877 | struct folio *folio, *fw_folio;
|
| 2878 | struct vm_area_struct *vma;
|
| 2879 | struct folio_walk fw;
|
| 2880 | struct page *page;
|
| 2881 | swp_entry_t entry;
|
| 2882 | pte_t swp_pte;
|
| 2883 | int ret;
|
| 2884 |
|
| 2885 | mmap_assert_locked(mm);
|
| 2886 | addr = PAGE_ALIGN_DOWN(addr);
|
| 2887 |
|
| 2888 | |
| 2889 | * Fault in the page writable and try to lock it; note that if the
|
| 2890 | * address would already be marked for exclusive use by a device,
|
| 2891 | * the GUP call would undo that first by triggering a fault.
|
| 2892 | *
|
| 2893 | * If any other device would already map this page exclusively, the
|
| 2894 | * fault will trigger a conversion to an ordinary
|
| 2895 | * (non-device-exclusive) PTE and issue a MMU_NOTIFY_EXCLUSIVE.
|
| 2896 |
|
| 2897 | retry:
|
| 2898 | page = get_user_page_vma_remote(mm, addr,
|
| 2899 | FOLL_GET | FOLL_WRITE | FOLL_SPLIT_PMD,
|
| 2900 | &vma);
|
| 2901 | if (IS_ERR(page))
|
| 2902 | return page;
|
| 2903 | folio = page_folio(page);
|
| 2904 |
|
| 2905 | if (!folio_test_anon(folio) || folio_test_hugetlb(folio)) {
|
| 2906 | folio_put(folio);
|
| 2907 | return ERR_PTR(-EOPNOTSUPP);
|
| 2908 | }
|
| 2909 |
|
| 2910 | ret = folio_lock_killable(folio);
|
| 2911 | if (ret) {
|
| 2912 | folio_put(folio);
|
| 2913 | return ERR_PTR(ret);
|
| 2914 | }
|
| 2915 |
|
| 2916 | |
| 2917 | * Inform secondary MMUs that we are going to convert this PTE to
|
| 2918 | * device-exclusive, such that they unmap it now. Note that the
|
| 2919 | * caller must filter this event out to prevent livelocks.
|
| 2920 |
|
| 2921 | mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
|
| 2922 | mm, addr, addr + PAGE_SIZE, owner);
|
| 2923 | mmu_notifier_invalidate_range_start(&range);
|
| 2924 |
|
| 2925 | |
| 2926 | * Let's do a second walk and make sure we still find the same page
|
| 2927 | * mapped writable. Note that any page of an anonymous folio can
|
| 2928 | * only be mapped writable using exactly one PTE ("exclusive"), so
|
| 2929 | * there cannot be other mappings.
|
| 2930 |
|
| 2931 | fw_folio = folio_walk_start(&fw, vma, addr, 0);
|
| 2932 | if (fw_folio != folio || fw.page != page ||
|
| 2933 | fw.level != FW_LEVEL_PTE || !pte_write(fw.pte)) {
|
| 2934 | if (fw_folio)
|
| 2935 | folio_walk_end(&fw, vma);
|
| 2936 | mmu_notifier_invalidate_range_end(&range);
|
| 2937 | folio_unlock(folio);
|
| 2938 | folio_put(folio);
|
| 2939 | goto retry;
|
| 2940 | }
|
| 2941 |
|
| 2942 |
|
| 2943 | flush_cache_page(vma, addr, page_to_pfn(page));
|
| 2944 | fw.pte = ptep_clear_flush(vma, addr, fw.ptep);
|
| 2945 |
|
| 2946 |
|
| 2947 | if (pte_dirty(fw.pte))
|
| 2948 | folio_mark_dirty(folio);
|
| 2949 |
|
| 2950 | |
| 2951 | * Store the pfn of the page in a special device-exclusive PFN swap PTE.
|
| 2952 | * do_swap_page() will trigger the conversion back while holding the
|
| 2953 | * folio lock.
|
| 2954 |
|
| 2955 | entry = make_device_exclusive_entry(page_to_pfn(page));
|
| 2956 | swp_pte = swp_entry_to_pte(entry);
|
| 2957 | if (pte_soft_dirty(fw.pte))
|
| 2958 | swp_pte = pte_swp_mksoft_dirty(swp_pte);
|
| 2959 |
|
| 2960 | set_pte_at(mm, addr, fw.ptep, swp_pte);
|
| 2961 |
|
| 2962 | folio_walk_end(&fw, vma);
|
| 2963 | mmu_notifier_invalidate_range_end(&range);
|
| 2964 | *foliop = folio;
|
| 2965 | return page;
|
| 2966 | }
|
| 2967 | EXPORT_SYMBOL_GPL(make_device_exclusive);
|
| 2968 | #endif
|
| 2969 |
|
| 2970 | void __put_anon_vma(struct anon_vma *anon_vma)
|
| 2971 | {
|
| 2972 | struct anon_vma *root = anon_vma->root;
|
| 2973 |
|
| 2974 | anon_vma_free(anon_vma);
|
| 2975 | if (root != anon_vma && atomic_dec_and_test(&root->refcount))
|
| 2976 | anon_vma_free(root);
|
| 2977 | }
|
| 2978 |
|
| 2979 | static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
|
| 2980 | struct rmap_walk_control *rwc)
|
| 2981 | {
|
| 2982 | struct anon_vma *anon_vma;
|
| 2983 |
|
| 2984 | if (rwc->anon_lock)
|
| 2985 | return rwc->anon_lock(folio, rwc);
|
| 2986 |
|
| 2987 | |
| 2988 | * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
|
| 2989 | * because that depends on folio_mapped(); but not all its usages
|
| 2990 | * are holding mmap_lock. Users without mmap_lock are required to
|
| 2991 | * take a reference count to prevent the anon_vma disappearing
|
| 2992 |
|
| 2993 | anon_vma = folio_anon_vma(folio);
|
| 2994 | if (!anon_vma)
|
| 2995 | return NULL;
|
| 2996 |
|
| 2997 | if (anon_vma_trylock_read(anon_vma))
|
| 2998 | goto out;
|
| 2999 |
|
| 3000 | if (rwc->try_lock) {
|
| 3001 | anon_vma = NULL;
|
| 3002 | rwc->contended = true;
|
| 3003 | goto out;
|
| 3004 | }
|
| 3005 |
|
| 3006 | anon_vma_lock_read(anon_vma);
|
| 3007 | out:
|
| 3008 | return anon_vma;
|
| 3009 | }
|
| 3010 |
|
| 3011 | |
| 3012 | * rmap_walk_anon - do something to anonymous page using the object-based
|
| 3013 | * rmap method
|
| 3014 | * @folio: the folio to be handled
|
| 3015 | * @rwc: control variable according to each walk type
|
| 3016 | * @locked: caller holds relevant rmap lock
|
| 3017 | *
|
| 3018 | * Find all the mappings of a folio using the mapping pointer and the vma
|
| 3019 | * chains contained in the anon_vma struct it points to.
|
| 3020 |
|
| 3021 | static void rmap_walk_anon(struct folio *folio,
|
| 3022 | struct rmap_walk_control *rwc, bool locked)
|
| 3023 | {
|
| 3024 | struct anon_vma *anon_vma;
|
| 3025 | pgoff_t pgoff_start, pgoff_end;
|
| 3026 | struct anon_vma_chain *avc;
|
| 3027 |
|
| 3028 | |
| 3029 | * The folio lock ensures that folio->mapping can't be changed under us
|
| 3030 | * to an anon_vma with different root.
|
| 3031 |
|
| 3032 | VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 3033 |
|
| 3034 | if (locked) {
|
| 3035 | anon_vma = folio_anon_vma(folio);
|
| 3036 |
|
| 3037 | VM_BUG_ON_FOLIO(!anon_vma, folio);
|
| 3038 | } else {
|
| 3039 | anon_vma = rmap_walk_anon_lock(folio, rwc);
|
| 3040 | }
|
| 3041 | if (!anon_vma)
|
| 3042 | return;
|
| 3043 |
|
| 3044 | pgoff_start = folio_pgoff(folio);
|
| 3045 | pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
|
| 3046 | anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) {
|
| 3047 | struct vm_area_struct *vma = avc->vma;
|
| 3048 | const unsigned long address = vma_anon_address(vma, pgoff_start,
|
| 3049 | folio_nr_pages(folio));
|
| 3050 |
|
| 3051 | VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma);
|
| 3052 | cond_resched();
|
| 3053 |
|
| 3054 | if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
|
| 3055 | continue;
|
| 3056 |
|
| 3057 | if (!rwc->rmap_one(folio, vma, address, rwc->arg))
|
| 3058 | break;
|
| 3059 | if (rwc->done && rwc->done(folio))
|
| 3060 | break;
|
| 3061 | }
|
| 3062 |
|
| 3063 | if (!locked)
|
| 3064 | anon_vma_unlock_read(anon_vma);
|
| 3065 | }
|
| 3066 |
|
| 3067 | |
| 3068 | * __rmap_walk_file() - Traverse the reverse mapping for a file-backed mapping
|
| 3069 | * of a page mapped within a specified page cache object at a specified offset.
|
| 3070 | *
|
| 3071 | * @folio: Either the folio whose mappings to traverse, or if NULL,
|
| 3072 | * the callbacks specified in @rwc will be configured such
|
| 3073 | * as to be able to look up mappings correctly.
|
| 3074 | * @mapping: The page cache object whose mapping VMAs we intend to
|
| 3075 | * traverse. If @folio is non-NULL, this should be equal to
|
| 3076 | * folio_mapping(folio).
|
| 3077 | * @pgoff_start: The offset within @mapping of the page which we are
|
| 3078 | * looking up. If @folio is non-NULL, this should be equal
|
| 3079 | * to folio_pgoff(folio).
|
| 3080 | * @nr_pages: The number of pages mapped by the mapping. If @folio is
|
| 3081 | * non-NULL, this should be equal to folio_nr_pages(folio).
|
| 3082 | * @rwc: The reverse mapping walk control object describing how
|
| 3083 | * the traversal should proceed.
|
| 3084 | * @locked: Is the @mapping already locked? If not, we acquire the
|
| 3085 | * lock.
|
| 3086 |
|
| 3087 | static void __rmap_walk_file(struct folio *folio, struct address_space *mapping,
|
| 3088 | pgoff_t pgoff_start, unsigned long nr_pages,
|
| 3089 | struct rmap_walk_control *rwc, bool locked)
|
| 3090 | {
|
| 3091 | pgoff_t pgoff_end = pgoff_start + nr_pages - 1;
|
| 3092 | struct vm_area_struct *vma;
|
| 3093 |
|
| 3094 | VM_WARN_ON_FOLIO(folio && mapping != folio_mapping(folio), folio);
|
| 3095 | VM_WARN_ON_FOLIO(folio && pgoff_start != folio_pgoff(folio), folio);
|
| 3096 | VM_WARN_ON_FOLIO(folio && nr_pages != folio_nr_pages(folio), folio);
|
| 3097 |
|
| 3098 | if (!locked) {
|
| 3099 | if (i_mmap_trylock_read(mapping))
|
| 3100 | goto lookup;
|
| 3101 |
|
| 3102 | if (rwc->try_lock) {
|
| 3103 | rwc->contended = true;
|
| 3104 | return;
|
| 3105 | }
|
| 3106 |
|
| 3107 | i_mmap_lock_read(mapping);
|
| 3108 | }
|
| 3109 | lookup:
|
| 3110 | mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) {
|
| 3111 | unsigned long address = vma_filebacked_address(vma, pgoff_start,
|
| 3112 | nr_pages);
|
| 3113 |
|
| 3114 | VM_BUG_ON_VMA(address == -EFAULT, vma);
|
| 3115 | cond_resched();
|
| 3116 |
|
| 3117 | if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
|
| 3118 | continue;
|
| 3119 |
|
| 3120 | if (!rwc->rmap_one(folio, vma, address, rwc->arg))
|
| 3121 | goto done;
|
| 3122 | if (rwc->done && rwc->done(folio))
|
| 3123 | goto done;
|
| 3124 | }
|
| 3125 | done:
|
| 3126 | if (!locked)
|
| 3127 | i_mmap_unlock_read(mapping);
|
| 3128 | }
|
| 3129 |
|
| 3130 | |
| 3131 | * rmap_walk_file - do something to file page using the object-based rmap method
|
| 3132 | * @folio: the folio to be handled
|
| 3133 | * @rwc: control variable according to each walk type
|
| 3134 | * @locked: caller holds relevant rmap lock
|
| 3135 | *
|
| 3136 | * Find all the mappings of a folio using the mapping pointer and the vma chains
|
| 3137 | * contained in the address_space struct it points to.
|
| 3138 |
|
| 3139 | static void rmap_walk_file(struct folio *folio,
|
| 3140 | struct rmap_walk_control *rwc, bool locked)
|
| 3141 | {
|
| 3142 | |
| 3143 | * The folio lock not only makes sure that folio->mapping cannot
|
| 3144 | * suddenly be NULLified by truncation, it makes sure that the structure
|
| 3145 | * at mapping cannot be freed and reused yet, so we can safely take
|
| 3146 | * mapping->i_mmap_rwsem.
|
| 3147 |
|
| 3148 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
|
| 3149 |
|
| 3150 | if (!folio->mapping)
|
| 3151 | return;
|
| 3152 |
|
| 3153 | __rmap_walk_file(folio, folio->mapping, folio->index,
|
| 3154 | folio_nr_pages(folio), rwc, locked);
|
| 3155 | }
|
| 3156 |
|
| 3157 | void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc)
|
| 3158 | {
|
| 3159 | if (unlikely(folio_test_ksm(folio)))
|
| 3160 | rmap_walk_ksm(folio, rwc);
|
| 3161 | else if (folio_test_anon(folio))
|
| 3162 | rmap_walk_anon(folio, rwc, false);
|
| 3163 | else
|
| 3164 | rmap_walk_file(folio, rwc, false);
|
| 3165 | }
|
| 3166 |
|
| 3167 |
|
| 3168 | void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc)
|
| 3169 | {
|
| 3170 |
|
| 3171 | VM_BUG_ON_FOLIO(folio_test_ksm(folio), folio);
|
| 3172 | if (folio_test_anon(folio))
|
| 3173 | rmap_walk_anon(folio, rwc, true);
|
| 3174 | else
|
| 3175 | rmap_walk_file(folio, rwc, true);
|
| 3176 | }
|
| 3177 |
|
| 3178 | #ifdef CONFIG_HUGETLB_PAGE
|
| 3179 | |
| 3180 | * The following two functions are for anonymous (private mapped) hugepages.
|
| 3181 | * Unlike common anonymous pages, anonymous hugepages have no accounting code
|
| 3182 | * and no lru code, because we handle hugepages differently from common pages.
|
| 3183 |
|
| 3184 | void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma,
|
| 3185 | unsigned long address, rmap_t flags)
|
| 3186 | {
|
| 3187 | VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
|
| 3188 | VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
|
| 3189 |
|
| 3190 | atomic_inc(&folio->_entire_mapcount);
|
| 3191 | atomic_inc(&folio->_large_mapcount);
|
| 3192 | if (flags & RMAP_EXCLUSIVE)
|
| 3193 | SetPageAnonExclusive(&folio->page);
|
| 3194 | VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 &&
|
| 3195 | PageAnonExclusive(&folio->page), folio);
|
| 3196 | }
|
| 3197 |
|
| 3198 | void hugetlb_add_new_anon_rmap(struct folio *folio,
|
| 3199 | struct vm_area_struct *vma, unsigned long address)
|
| 3200 | {
|
| 3201 | VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
|
| 3202 |
|
| 3203 | BUG_ON(address < vma->vm_start || address >= vma->vm_end);
|
| 3204 |
|
| 3205 | atomic_set(&folio->_entire_mapcount, 0);
|
| 3206 | atomic_set(&folio->_large_mapcount, 0);
|
| 3207 | folio_clear_hugetlb_restore_reserve(folio);
|
| 3208 | __folio_set_anon(folio, vma, address, true);
|
| 3209 | SetPageAnonExclusive(&folio->page);
|
| 3210 | }
|
| 3211 | #endif
|