| 1 | |
| 2 | |
| 3 | * Memory Migration functionality - linux/mm/migrate.c
|
| 4 | *
|
| 5 | * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
|
| 6 | *
|
| 7 | * Page migration was first developed in the context of the memory hotplug
|
| 8 | * project. The main authors of the migration code are:
|
| 9 | *
|
| 10 | * IWAMOTO Toshihiro <[email protected]>
|
| 11 | * Hirokazu Takahashi <[email protected]>
|
| 12 | * Dave Hansen <[email protected]>
|
| 13 | * Christoph Lameter
|
| 14 |
|
| 15 |
|
| 16 | #include <linux/migrate.h>
|
| 17 | #include <linux/export.h>
|
| 18 | #include <linux/swap.h>
|
| 19 | #include <linux/leafops.h>
|
| 20 | #include <linux/pagemap.h>
|
| 21 | #include <linux/buffer_head.h>
|
| 22 | #include <linux/mm_inline.h>
|
| 23 | #include <linux/ksm.h>
|
| 24 | #include <linux/rmap.h>
|
| 25 | #include <linux/topology.h>
|
| 26 | #include <linux/cpu.h>
|
| 27 | #include <linux/cpuset.h>
|
| 28 | #include <linux/writeback.h>
|
| 29 | #include <linux/mempolicy.h>
|
| 30 | #include <linux/vmalloc.h>
|
| 31 | #include <linux/security.h>
|
| 32 | #include <linux/backing-dev.h>
|
| 33 | #include <linux/compaction.h>
|
| 34 | #include <linux/syscalls.h>
|
| 35 | #include <linux/compat.h>
|
| 36 | #include <linux/hugetlb.h>
|
| 37 | #include <linux/gfp.h>
|
| 38 | #include <linux/page_idle.h>
|
| 39 | #include <linux/page_owner.h>
|
| 40 | #include <linux/sched/mm.h>
|
| 41 | #include <linux/ptrace.h>
|
| 42 | #include <linux/memory.h>
|
| 43 | #include <linux/sched/sysctl.h>
|
| 44 | #include <linux/memory-tiers.h>
|
| 45 | #include <linux/pagewalk.h>
|
| 46 |
|
| 47 | #include <asm/tlbflush.h>
|
| 48 |
|
| 49 | #include <trace/events/migrate.h>
|
| 50 |
|
| 51 | #include "internal.h"
|
| 52 | #include "page_alloc.h"
|
| 53 | #include "swap.h"
|
| 54 |
|
| 55 | static const struct movable_operations *offline_movable_ops;
|
| 56 | static const struct movable_operations *zsmalloc_movable_ops;
|
| 57 |
|
| 58 | int set_movable_ops(const struct movable_operations *ops, enum pagetype type)
|
| 59 | {
|
| 60 | |
| 61 | * We only allow for selected types and don't handle concurrent
|
| 62 | * registration attempts yet.
|
| 63 |
|
| 64 | switch (type) {
|
| 65 | case PGTY_offline:
|
| 66 | if (offline_movable_ops && ops)
|
| 67 | return -EBUSY;
|
| 68 | offline_movable_ops = ops;
|
| 69 | break;
|
| 70 | case PGTY_zsmalloc:
|
| 71 | if (zsmalloc_movable_ops && ops)
|
| 72 | return -EBUSY;
|
| 73 | zsmalloc_movable_ops = ops;
|
| 74 | break;
|
| 75 | default:
|
| 76 | return -EINVAL;
|
| 77 | }
|
| 78 | return 0;
|
| 79 | }
|
| 80 | EXPORT_SYMBOL_GPL(set_movable_ops);
|
| 81 |
|
| 82 | static const struct movable_operations *page_movable_ops(struct page *page)
|
| 83 | {
|
| 84 | VM_WARN_ON_ONCE_PAGE(!page_has_movable_ops(page), page);
|
| 85 |
|
| 86 | |
| 87 | * If we enable page migration for a page of a certain type by marking
|
| 88 | * it as movable, the page type must be sticky until the page gets freed
|
| 89 | * back to the buddy.
|
| 90 |
|
| 91 | if (PageOffline(page))
|
| 92 |
|
| 93 | return offline_movable_ops;
|
| 94 | if (PageZsmalloc(page))
|
| 95 | return zsmalloc_movable_ops;
|
| 96 |
|
| 97 | return NULL;
|
| 98 | }
|
| 99 |
|
| 100 | |
| 101 | * isolate_movable_ops_page - isolate a movable_ops page for migration
|
| 102 | * @page: The page.
|
| 103 | * @mode: The isolation mode.
|
| 104 | *
|
| 105 | * Try to isolate a movable_ops page for migration. Will fail if the page is
|
| 106 | * not a movable_ops page, if the page is already isolated for migration
|
| 107 | * or if the page was just was released by its owner.
|
| 108 | *
|
| 109 | * Once isolated, the page cannot get freed until it is either putback
|
| 110 | * or migrated.
|
| 111 | *
|
| 112 | * Returns true if isolation succeeded, otherwise false.
|
| 113 |
|
| 114 | bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode)
|
| 115 | {
|
| 116 | |
| 117 | * TODO: these pages will not be folios in the future. All
|
| 118 | * folio dependencies will have to be removed.
|
| 119 |
|
| 120 | struct folio *folio = folio_get_nontail_page(page);
|
| 121 | const struct movable_operations *mops;
|
| 122 |
|
| 123 | |
| 124 | * Avoid burning cycles with pages that are yet under __free_pages(),
|
| 125 | * or just got freed under us.
|
| 126 | *
|
| 127 | * In case we 'win' a race for a movable page being freed under us and
|
| 128 | * raise its refcount preventing __free_pages() from doing its job
|
| 129 | * the put_page() at the end of this block will take care of
|
| 130 | * release this page, thus avoiding a nasty leakage.
|
| 131 |
|
| 132 | if (!folio)
|
| 133 | goto out;
|
| 134 |
|
| 135 | |
| 136 | * Check for movable_ops pages before taking the page lock because
|
| 137 | * we use non-atomic bitops on newly allocated page flags so
|
| 138 | * unconditionally grabbing the lock ruins page's owner side.
|
| 139 | *
|
| 140 | * Note that once a page has movable_ops, it will stay that way
|
| 141 | * until the page was freed.
|
| 142 |
|
| 143 | if (unlikely(!page_has_movable_ops(page)))
|
| 144 | goto out_putfolio;
|
| 145 |
|
| 146 | |
| 147 | * As movable pages are not isolated from LRU lists, concurrent
|
| 148 | * compaction threads can race against page migration functions
|
| 149 | * as well as race against the releasing a page.
|
| 150 | *
|
| 151 | * In order to avoid having an already isolated movable page
|
| 152 | * being (wrongly) re-isolated while it is under migration,
|
| 153 | * or to avoid attempting to isolate pages being released,
|
| 154 | * lets be sure we have the page lock
|
| 155 | * before proceeding with the movable page isolation steps.
|
| 156 |
|
| 157 | if (unlikely(!folio_trylock(folio)))
|
| 158 | goto out_putfolio;
|
| 159 |
|
| 160 | VM_WARN_ON_ONCE_PAGE(!page_has_movable_ops(page), page);
|
| 161 | if (PageMovableOpsIsolated(page))
|
| 162 | goto out_no_isolated;
|
| 163 |
|
| 164 | mops = page_movable_ops(page);
|
| 165 | if (WARN_ON_ONCE(!mops))
|
| 166 | goto out_no_isolated;
|
| 167 |
|
| 168 | if (!mops->isolate_page(page, mode))
|
| 169 | goto out_no_isolated;
|
| 170 |
|
| 171 |
|
| 172 | VM_WARN_ON_ONCE_PAGE(PageMovableOpsIsolated(page), page);
|
| 173 | SetPageMovableOpsIsolated(page);
|
| 174 | folio_unlock(folio);
|
| 175 |
|
| 176 | return true;
|
| 177 |
|
| 178 | out_no_isolated:
|
| 179 | folio_unlock(folio);
|
| 180 | out_putfolio:
|
| 181 | folio_put(folio);
|
| 182 | out:
|
| 183 | return false;
|
| 184 | }
|
| 185 |
|
| 186 | |
| 187 | * putback_movable_ops_page - putback an isolated movable_ops page
|
| 188 | * @page: The isolated page.
|
| 189 | *
|
| 190 | * Putback an isolated movable_ops page.
|
| 191 | *
|
| 192 | * After the page was putback, it might get freed instantly.
|
| 193 |
|
| 194 | static void putback_movable_ops_page(struct page *page)
|
| 195 | {
|
| 196 | |
| 197 | * TODO: these pages will not be folios in the future. All
|
| 198 | * folio dependencies will have to be removed.
|
| 199 |
|
| 200 | struct folio *folio = page_folio(page);
|
| 201 |
|
| 202 | VM_WARN_ON_ONCE_PAGE(!page_has_movable_ops(page), page);
|
| 203 | VM_WARN_ON_ONCE_PAGE(!PageMovableOpsIsolated(page), page);
|
| 204 | folio_lock(folio);
|
| 205 | page_movable_ops(page)->putback_page(page);
|
| 206 | ClearPageMovableOpsIsolated(page);
|
| 207 | folio_unlock(folio);
|
| 208 | folio_put(folio);
|
| 209 | }
|
| 210 |
|
| 211 | |
| 212 | * migrate_movable_ops_page - migrate an isolated movable_ops page
|
| 213 | * @dst: The destination page.
|
| 214 | * @src: The source page.
|
| 215 | * @mode: The migration mode.
|
| 216 | *
|
| 217 | * Migrate an isolated movable_ops page.
|
| 218 | *
|
| 219 | * If the src page was already released by its owner, the src page is
|
| 220 | * un-isolated (putback) and migration succeeds; the migration core will be the
|
| 221 | * owner of both pages.
|
| 222 | *
|
| 223 | * If the src page was not released by its owner and the migration was
|
| 224 | * successful, the owner of the src page and the dst page are swapped and
|
| 225 | * the src page is un-isolated.
|
| 226 | *
|
| 227 | * If migration fails, the ownership stays unmodified and the src page
|
| 228 | * remains isolated: migration may be retried later or the page can be putback.
|
| 229 | *
|
| 230 | * TODO: migration core will treat both pages as folios and lock them before
|
| 231 | * this call to unlock them after this call. Further, the folio refcounts on
|
| 232 | * src and dst are also released by migration core. These pages will not be
|
| 233 | * folios in the future, so that must be reworked.
|
| 234 | *
|
| 235 | * Returns 0 on success, otherwise a negative error code.
|
| 236 |
|
| 237 | static int migrate_movable_ops_page(struct page *dst, struct page *src,
|
| 238 | enum migrate_mode mode)
|
| 239 | {
|
| 240 | int rc;
|
| 241 |
|
| 242 | VM_WARN_ON_ONCE_PAGE(!page_has_movable_ops(src), src);
|
| 243 | VM_WARN_ON_ONCE_PAGE(!PageMovableOpsIsolated(src), src);
|
| 244 | rc = page_movable_ops(src)->migrate_page(dst, src, mode);
|
| 245 | if (!rc)
|
| 246 | ClearPageMovableOpsIsolated(src);
|
| 247 | return rc;
|
| 248 | }
|
| 249 |
|
| 250 | |
| 251 | * Put previously isolated pages back onto the appropriate lists
|
| 252 | * from where they were once taken off for compaction/migration.
|
| 253 | *
|
| 254 | * This function shall be used whenever the isolated pageset has been
|
| 255 | * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
|
| 256 | * and folio_isolate_hugetlb().
|
| 257 |
|
| 258 | void putback_movable_pages(struct list_head *l)
|
| 259 | {
|
| 260 | struct folio *folio;
|
| 261 | struct folio *folio2;
|
| 262 |
|
| 263 | list_for_each_entry_safe(folio, folio2, l, lru) {
|
| 264 | if (unlikely(folio_test_hugetlb(folio))) {
|
| 265 | folio_putback_hugetlb(folio);
|
| 266 | continue;
|
| 267 | }
|
| 268 | list_del(&folio->lru);
|
| 269 | if (unlikely(page_has_movable_ops(&folio->page))) {
|
| 270 | putback_movable_ops_page(&folio->page);
|
| 271 | } else {
|
| 272 | node_stat_mod_folio(folio, NR_ISOLATED_ANON +
|
| 273 | folio_is_file_lru(folio), -folio_nr_pages(folio));
|
| 274 | folio_putback_lru(folio);
|
| 275 | }
|
| 276 | }
|
| 277 | }
|
| 278 |
|
| 279 |
|
| 280 | bool isolate_folio_to_list(struct folio *folio, struct list_head *list)
|
| 281 | {
|
| 282 | if (folio_test_hugetlb(folio))
|
| 283 | return folio_isolate_hugetlb(folio, list);
|
| 284 |
|
| 285 | if (page_has_movable_ops(&folio->page)) {
|
| 286 | if (!isolate_movable_ops_page(&folio->page,
|
| 287 | ISOLATE_UNEVICTABLE))
|
| 288 | return false;
|
| 289 | } else {
|
| 290 | if (!folio_isolate_lru(folio))
|
| 291 | return false;
|
| 292 | node_stat_add_folio(folio, NR_ISOLATED_ANON +
|
| 293 | folio_is_file_lru(folio));
|
| 294 | }
|
| 295 | list_add(&folio->lru, list);
|
| 296 | return true;
|
| 297 | }
|
| 298 |
|
| 299 | static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw,
|
| 300 | struct folio *folio, pte_t old_pte, unsigned long idx)
|
| 301 | {
|
| 302 | struct page *page = folio_page(folio, idx);
|
| 303 | pte_t newpte;
|
| 304 |
|
| 305 | if (PageCompound(page) || PageHWPoison(page))
|
| 306 | return false;
|
| 307 |
|
| 308 | VM_BUG_ON_PAGE(!PageAnon(page), page);
|
| 309 | VM_BUG_ON_PAGE(!PageLocked(page), page);
|
| 310 | VM_BUG_ON_PAGE(pte_present(old_pte), page);
|
| 311 | VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio);
|
| 312 |
|
| 313 | if (folio_test_mlocked(folio) || (pvmw->vma->vm_flags & VM_LOCKED) ||
|
| 314 | mm_forbids_zeropage(pvmw->vma->vm_mm))
|
| 315 | return false;
|
| 316 |
|
| 317 | |
| 318 | * The pmd entry mapping the old thp was flushed and the pte mapping
|
| 319 | * this subpage has been non present. If the subpage is only zero-filled
|
| 320 | * then map it to the shared zeropage.
|
| 321 |
|
| 322 | if (!pages_identical(page, ZERO_PAGE(0)))
|
| 323 | return false;
|
| 324 |
|
| 325 | newpte = pte_mkspecial(pfn_pte(zero_pfn(pvmw->address),
|
| 326 | pvmw->vma->vm_page_prot));
|
| 327 |
|
| 328 | if (pte_swp_soft_dirty(old_pte))
|
| 329 | newpte = pte_mksoft_dirty(newpte);
|
| 330 | if (pte_swp_uffd(old_pte))
|
| 331 | newpte = pte_mkuffd(newpte);
|
| 332 |
|
| 333 |
|
| 334 | if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma))
|
| 335 | newpte = pte_modify(newpte, PAGE_NONE);
|
| 336 |
|
| 337 | set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte);
|
| 338 |
|
| 339 | dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio));
|
| 340 | return true;
|
| 341 | }
|
| 342 |
|
| 343 | struct rmap_walk_arg {
|
| 344 | struct folio *folio;
|
| 345 | bool map_unused_to_zeropage;
|
| 346 | };
|
| 347 |
|
| 348 | |
| 349 | * Restore a potential migration pte to a working pte entry
|
| 350 |
|
| 351 | static bool remove_migration_pte(struct folio *folio,
|
| 352 | struct vm_area_struct *vma, unsigned long addr, void *arg)
|
| 353 | {
|
| 354 | struct rmap_walk_arg *rmap_walk_arg = arg;
|
| 355 | DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
|
| 356 |
|
| 357 | while (page_vma_mapped_walk(&pvmw)) {
|
| 358 | rmap_t rmap_flags = RMAP_NONE;
|
| 359 | unsigned long idx = 0;
|
| 360 | softleaf_t entry;
|
| 361 | struct page *new;
|
| 362 | pte_t old_pte;
|
| 363 | pte_t pte;
|
| 364 |
|
| 365 | #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
|
| 366 |
|
| 367 | if (!pvmw.pte) {
|
| 368 | VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
|
| 369 | !folio_test_pmd_mappable(folio), folio);
|
| 370 | remove_migration_pmd(&pvmw, folio);
|
| 371 | continue;
|
| 372 | }
|
| 373 | #endif
|
| 374 | if (folio_test_hugetlb(folio))
|
| 375 | old_pte = huge_ptep_get(vma->vm_mm, pvmw.address,
|
| 376 | pvmw.pte);
|
| 377 | else
|
| 378 | old_pte = ptep_get(pvmw.pte);
|
| 379 |
|
| 380 | entry = softleaf_from_pte(old_pte);
|
| 381 | if (folio_test_large(folio) && !folio_test_hugetlb(folio))
|
| 382 | idx = softleaf_to_pfn(entry) - pvmw.pfn;
|
| 383 |
|
| 384 | if (rmap_walk_arg->map_unused_to_zeropage &&
|
| 385 | try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
|
| 386 | continue;
|
| 387 |
|
| 388 | folio_get(folio);
|
| 389 | new = folio_page(folio, idx);
|
| 390 | pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
|
| 391 | if (!softleaf_is_migration_young(entry))
|
| 392 | pte = pte_mkold(pte);
|
| 393 | if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry))
|
| 394 | pte = pte_mkdirty(pte);
|
| 395 | if (pte_swp_soft_dirty(old_pte))
|
| 396 | pte = pte_mksoft_dirty(pte);
|
| 397 | else
|
| 398 | pte = pte_clear_soft_dirty(pte);
|
| 399 |
|
| 400 | if (softleaf_is_migration_write(entry))
|
| 401 | pte = pte_mkwrite(pte, vma);
|
| 402 | else if (pte_swp_uffd(old_pte))
|
| 403 | pte = pte_mkuffd(pte);
|
| 404 |
|
| 405 |
|
| 406 | if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
|
| 407 | pte = pte_modify(pte, PAGE_NONE);
|
| 408 |
|
| 409 | if (folio_test_anon(folio) && !softleaf_is_migration_read(entry))
|
| 410 | rmap_flags |= RMAP_EXCLUSIVE;
|
| 411 |
|
| 412 | if (unlikely(is_device_private_page(new))) {
|
| 413 | if (pte_write(pte))
|
| 414 | entry = make_writable_device_private_entry(
|
| 415 | page_to_pfn(new));
|
| 416 | else
|
| 417 | entry = make_readable_device_private_entry(
|
| 418 | page_to_pfn(new));
|
| 419 | pte = softleaf_to_pte(entry);
|
| 420 | if (pte_swp_soft_dirty(old_pte))
|
| 421 | pte = pte_swp_mksoft_dirty(pte);
|
| 422 | if (pte_swp_uffd(old_pte))
|
| 423 | pte = pte_swp_mkuffd(pte);
|
| 424 | }
|
| 425 |
|
| 426 | #ifdef CONFIG_HUGETLB_PAGE
|
| 427 | if (folio_test_hugetlb(folio)) {
|
| 428 | struct hstate *h = hstate_vma(vma);
|
| 429 | unsigned int shift = huge_page_shift(h);
|
| 430 | unsigned long psize = huge_page_size(h);
|
| 431 |
|
| 432 | pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
|
| 433 | if (folio_test_anon(folio))
|
| 434 | hugetlb_add_anon_rmap(folio, vma, pvmw.address,
|
| 435 | rmap_flags);
|
| 436 | else
|
| 437 | hugetlb_add_file_rmap(folio);
|
| 438 | set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte,
|
| 439 | psize);
|
| 440 | } else
|
| 441 | #endif
|
| 442 | {
|
| 443 | if (folio_test_anon(folio))
|
| 444 | folio_add_anon_rmap_pte(folio, new, vma,
|
| 445 | pvmw.address, rmap_flags);
|
| 446 | else
|
| 447 | folio_add_file_rmap_pte(folio, new, vma);
|
| 448 | set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
|
| 449 | }
|
| 450 | if (READ_ONCE(vma->vm_flags) & VM_LOCKED)
|
| 451 | mlock_drain_local();
|
| 452 |
|
| 453 | trace_remove_migration_pte(pvmw.address, pte_val(pte),
|
| 454 | compound_order(new));
|
| 455 |
|
| 456 |
|
| 457 | update_mmu_cache(vma, pvmw.address, pvmw.pte);
|
| 458 | }
|
| 459 |
|
| 460 | return true;
|
| 461 | }
|
| 462 |
|
| 463 | |
| 464 | * Get rid of all migration entries and replace them by
|
| 465 | * references to the indicated page.
|
| 466 |
|
| 467 | void remove_migration_ptes(struct folio *src, struct folio *dst,
|
| 468 | enum ttu_flags flags)
|
| 469 | {
|
| 470 | struct rmap_walk_arg rmap_walk_arg = {
|
| 471 | .folio = src,
|
| 472 | .map_unused_to_zeropage = flags & TTU_USE_SHARED_ZEROPAGE,
|
| 473 | };
|
| 474 |
|
| 475 | struct rmap_walk_control rwc = {
|
| 476 | .rmap_one = remove_migration_pte,
|
| 477 | .arg = &rmap_walk_arg,
|
| 478 | };
|
| 479 |
|
| 480 | VM_BUG_ON_FOLIO((flags & TTU_USE_SHARED_ZEROPAGE) && (src != dst), src);
|
| 481 |
|
| 482 | if (flags & TTU_RMAP_LOCKED)
|
| 483 | rmap_walk_locked(dst, &rwc);
|
| 484 | else
|
| 485 | rmap_walk(dst, &rwc);
|
| 486 | }
|
| 487 |
|
| 488 | |
| 489 | * Something used the pte of a page under migration. We need to
|
| 490 | * get to the page and wait until migration is finished.
|
| 491 | * When we return from this function the fault will be retried.
|
| 492 |
|
| 493 | void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
|
| 494 | unsigned long address)
|
| 495 | {
|
| 496 | spinlock_t *ptl;
|
| 497 | pte_t *ptep;
|
| 498 | pte_t pte;
|
| 499 | softleaf_t entry;
|
| 500 |
|
| 501 | ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
|
| 502 | if (!ptep)
|
| 503 | return;
|
| 504 |
|
| 505 | pte = ptep_get(ptep);
|
| 506 | pte_unmap(ptep);
|
| 507 |
|
| 508 | if (pte_none(pte) || pte_present(pte))
|
| 509 | goto out;
|
| 510 |
|
| 511 | entry = softleaf_from_pte(pte);
|
| 512 | if (!softleaf_is_migration(entry))
|
| 513 | goto out;
|
| 514 |
|
| 515 | softleaf_entry_wait_on_locked(entry, ptl);
|
| 516 | return;
|
| 517 | out:
|
| 518 | spin_unlock(ptl);
|
| 519 | }
|
| 520 |
|
| 521 | #ifdef CONFIG_HUGETLB_PAGE
|
| 522 | |
| 523 | * The vma read lock must be held upon entry. Holding that lock prevents either
|
| 524 | * the pte or the ptl from being freed.
|
| 525 | *
|
| 526 | * This function will release the vma lock before returning.
|
| 527 |
|
| 528 | void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
|
| 529 | {
|
| 530 | spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, ptep);
|
| 531 | softleaf_t entry;
|
| 532 | pte_t pte;
|
| 533 |
|
| 534 | hugetlb_vma_assert_locked(vma);
|
| 535 | spin_lock(ptl);
|
| 536 | pte = huge_ptep_get(vma->vm_mm, addr, ptep);
|
| 537 |
|
| 538 | if (huge_pte_none(pte))
|
| 539 | goto fail;
|
| 540 |
|
| 541 | entry = softleaf_from_pte(pte);
|
| 542 | if (softleaf_is_migration(entry)) {
|
| 543 | |
| 544 | * If migration entry existed, safe to release vma lock
|
| 545 | * here because the pgtable page won't be freed without the
|
| 546 | * pgtable lock released. See comment right above pgtable
|
| 547 | * lock release in softleaf_entry_wait_on_locked().
|
| 548 |
|
| 549 | hugetlb_vma_unlock_read(vma);
|
| 550 | softleaf_entry_wait_on_locked(entry, ptl);
|
| 551 | return;
|
| 552 | }
|
| 553 |
|
| 554 | fail:
|
| 555 | spin_unlock(ptl);
|
| 556 | hugetlb_vma_unlock_read(vma);
|
| 557 | }
|
| 558 | #endif
|
| 559 |
|
| 560 | #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
|
| 561 | void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
|
| 562 | {
|
| 563 | spinlock_t *ptl;
|
| 564 |
|
| 565 | ptl = pmd_lock(mm, pmd);
|
| 566 | if (!pmd_is_migration_entry(*pmd))
|
| 567 | goto unlock;
|
| 568 | softleaf_entry_wait_on_locked(softleaf_from_pmd(*pmd), ptl);
|
| 569 | return;
|
| 570 | unlock:
|
| 571 | spin_unlock(ptl);
|
| 572 | }
|
| 573 | #endif
|
| 574 |
|
| 575 | |
| 576 | * Replace the folio in the mapping.
|
| 577 | *
|
| 578 | * The number of remaining references must be:
|
| 579 | * 1 for anonymous folios without a mapping
|
| 580 | * 2 for folios with a mapping
|
| 581 | * 3 for folios with a mapping and the private flag set.
|
| 582 |
|
| 583 | static int __folio_migrate_mapping(struct address_space *mapping,
|
| 584 | struct folio *newfolio, struct folio *folio, int expected_count)
|
| 585 | {
|
| 586 | XA_STATE(xas, &mapping->i_pages, folio->index);
|
| 587 | struct swap_cluster_info *ci = NULL;
|
| 588 | struct zone *oldzone, *newzone;
|
| 589 | int dirty;
|
| 590 | long nr = folio_nr_pages(folio);
|
| 591 |
|
| 592 | if (!mapping) {
|
| 593 |
|
| 594 | if (folio_test_large(folio) &&
|
| 595 | folio_test_large_rmappable(folio)) {
|
| 596 | if (!folio_ref_freeze(folio, expected_count))
|
| 597 | return -EAGAIN;
|
| 598 | folio_unqueue_deferred_split(folio);
|
| 599 | folio_ref_unfreeze(folio, expected_count);
|
| 600 | }
|
| 601 |
|
| 602 |
|
| 603 | newfolio->index = folio->index;
|
| 604 | newfolio->mapping = folio->mapping;
|
| 605 | if (folio_test_anon(folio) && folio_test_large(folio) &&
|
| 606 | !folio_test_hugetlb(folio))
|
| 607 | mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
|
| 608 | if (folio_test_swapbacked(folio))
|
| 609 | __folio_set_swapbacked(newfolio);
|
| 610 |
|
| 611 | return 0;
|
| 612 | }
|
| 613 |
|
| 614 | oldzone = folio_zone(folio);
|
| 615 | newzone = folio_zone(newfolio);
|
| 616 |
|
| 617 | if (folio_test_swapcache(folio))
|
| 618 | ci = swap_cluster_get_and_lock_irq(folio);
|
| 619 | else
|
| 620 | xas_lock_irq(&xas);
|
| 621 |
|
| 622 | if (!folio_ref_freeze(folio, expected_count)) {
|
| 623 | if (ci)
|
| 624 | swap_cluster_unlock_irq(ci);
|
| 625 | else
|
| 626 | xas_unlock_irq(&xas);
|
| 627 | return -EAGAIN;
|
| 628 | }
|
| 629 |
|
| 630 |
|
| 631 | folio_unqueue_deferred_split(folio);
|
| 632 |
|
| 633 | |
| 634 | * Now we know that no one else is looking at the folio:
|
| 635 | * no turning back from here.
|
| 636 |
|
| 637 | newfolio->index = folio->index;
|
| 638 | newfolio->mapping = folio->mapping;
|
| 639 | if (folio_test_anon(folio) && folio_test_large(folio))
|
| 640 | mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
|
| 641 | folio_ref_add(newfolio, nr);
|
| 642 | if (folio_test_swapbacked(folio))
|
| 643 | __folio_set_swapbacked(newfolio);
|
| 644 | if (folio_test_swapcache(folio)) {
|
| 645 | folio_set_swapcache(newfolio);
|
| 646 | newfolio->private = folio_get_private(folio);
|
| 647 | }
|
| 648 |
|
| 649 |
|
| 650 | dirty = folio_test_dirty(folio);
|
| 651 | if (dirty) {
|
| 652 | folio_clear_dirty(folio);
|
| 653 | folio_set_dirty(newfolio);
|
| 654 | }
|
| 655 |
|
| 656 | if (folio_test_swapcache(folio))
|
| 657 | __swap_cache_replace_folio(ci, folio, newfolio);
|
| 658 | else
|
| 659 | xas_store(&xas, newfolio);
|
| 660 |
|
| 661 | |
| 662 | * Drop cache reference from old folio by unfreezing
|
| 663 | * to one less reference.
|
| 664 | * We know this isn't the last reference.
|
| 665 |
|
| 666 | folio_ref_unfreeze(folio, expected_count - nr);
|
| 667 |
|
| 668 |
|
| 669 | if (ci)
|
| 670 | swap_cluster_unlock(ci);
|
| 671 | else
|
| 672 | xas_unlock(&xas);
|
| 673 |
|
| 674 | |
| 675 | * If moved to a different zone then also account
|
| 676 | * the folio for that zone. Other VM counters will be
|
| 677 | * taken care of when we establish references to the
|
| 678 | * new folio and drop references to the old folio.
|
| 679 | *
|
| 680 | * Note that anonymous folios are accounted for
|
| 681 | * via NR_FILE_PAGES and NR_ANON_MAPPED if they
|
| 682 | * are mapped to swap space.
|
| 683 |
|
| 684 | if (newzone != oldzone) {
|
| 685 | struct lruvec *old_lruvec, *new_lruvec;
|
| 686 | struct mem_cgroup *memcg;
|
| 687 |
|
| 688 | rcu_read_lock();
|
| 689 | memcg = folio_memcg(folio);
|
| 690 | old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
|
| 691 | new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);
|
| 692 |
|
| 693 | mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
|
| 694 | mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
|
| 695 | if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) {
|
| 696 | mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
|
| 697 | mod_lruvec_state(new_lruvec, NR_SHMEM, nr);
|
| 698 |
|
| 699 | if (folio_test_pmd_mappable(folio)) {
|
| 700 | mod_lruvec_state(old_lruvec, NR_SHMEM_THPS, -nr);
|
| 701 | mod_lruvec_state(new_lruvec, NR_SHMEM_THPS, nr);
|
| 702 | }
|
| 703 | }
|
| 704 | #ifdef CONFIG_SWAP
|
| 705 | if (folio_test_swapcache(folio)) {
|
| 706 | mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr);
|
| 707 | mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr);
|
| 708 | }
|
| 709 | #endif
|
| 710 | if (dirty && mapping_can_writeback(mapping)) {
|
| 711 | mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
|
| 712 | __mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
|
| 713 | mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
|
| 714 | __mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
|
| 715 | }
|
| 716 | rcu_read_unlock();
|
| 717 | }
|
| 718 | local_irq_enable();
|
| 719 |
|
| 720 | return 0;
|
| 721 | }
|
| 722 |
|
| 723 | int folio_migrate_mapping(struct address_space *mapping,
|
| 724 | struct folio *newfolio, struct folio *folio, int extra_count)
|
| 725 | {
|
| 726 | int expected_count = folio_expected_ref_count(folio) + extra_count + 1;
|
| 727 |
|
| 728 | if (folio_ref_count(folio) != expected_count)
|
| 729 | return -EAGAIN;
|
| 730 |
|
| 731 | return __folio_migrate_mapping(mapping, newfolio, folio, expected_count);
|
| 732 | }
|
| 733 | EXPORT_SYMBOL(folio_migrate_mapping);
|
| 734 |
|
| 735 | |
| 736 | * The expected number of remaining references is the same as that
|
| 737 | * of folio_migrate_mapping().
|
| 738 |
|
| 739 | int migrate_huge_page_move_mapping(struct address_space *mapping,
|
| 740 | struct folio *dst, struct folio *src)
|
| 741 | {
|
| 742 | XA_STATE(xas, &mapping->i_pages, src->index);
|
| 743 | int rc, expected_count = folio_expected_ref_count(src) + 1;
|
| 744 |
|
| 745 | if (folio_ref_count(src) != expected_count)
|
| 746 | return -EAGAIN;
|
| 747 |
|
| 748 | rc = folio_mc_copy(dst, src);
|
| 749 | if (unlikely(rc))
|
| 750 | return rc;
|
| 751 |
|
| 752 | xas_lock_irq(&xas);
|
| 753 | if (!folio_ref_freeze(src, expected_count)) {
|
| 754 | xas_unlock_irq(&xas);
|
| 755 | return -EAGAIN;
|
| 756 | }
|
| 757 |
|
| 758 | dst->index = src->index;
|
| 759 | dst->mapping = src->mapping;
|
| 760 |
|
| 761 | folio_ref_add(dst, folio_nr_pages(dst));
|
| 762 |
|
| 763 | xas_store(&xas, dst);
|
| 764 |
|
| 765 | folio_ref_unfreeze(src, expected_count - folio_nr_pages(src));
|
| 766 |
|
| 767 | xas_unlock_irq(&xas);
|
| 768 |
|
| 769 | return 0;
|
| 770 | }
|
| 771 |
|
| 772 | |
| 773 | * Copy the flags and some other ancillary information
|
| 774 |
|
| 775 | void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
|
| 776 | {
|
| 777 | int cpupid;
|
| 778 |
|
| 779 | if (folio_test_referenced(folio))
|
| 780 | folio_set_referenced(newfolio);
|
| 781 | if (folio_test_uptodate(folio))
|
| 782 | folio_mark_uptodate(newfolio);
|
| 783 | if (folio_test_clear_active(folio)) {
|
| 784 | VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio);
|
| 785 | folio_set_active(newfolio);
|
| 786 | } else if (folio_test_clear_unevictable(folio))
|
| 787 | folio_set_unevictable(newfolio);
|
| 788 | if (folio_test_workingset(folio))
|
| 789 | folio_set_workingset(newfolio);
|
| 790 | if (folio_test_checked(folio))
|
| 791 | folio_set_checked(newfolio);
|
| 792 | |
| 793 | * PG_anon_exclusive (-> PG_mappedtodisk) is always migrated via
|
| 794 | * migration entries. We can still have PG_anon_exclusive set on an
|
| 795 | * effectively unmapped and unreferenced first sub-pages of an
|
| 796 | * anonymous THP: we can simply copy it here via PG_mappedtodisk.
|
| 797 |
|
| 798 | if (folio_test_mappedtodisk(folio))
|
| 799 | folio_set_mappedtodisk(newfolio);
|
| 800 |
|
| 801 |
|
| 802 | if (folio_test_dirty(folio))
|
| 803 | folio_set_dirty(newfolio);
|
| 804 |
|
| 805 | if (folio_test_young(folio))
|
| 806 | folio_set_young(newfolio);
|
| 807 | if (folio_test_idle(folio))
|
| 808 | folio_set_idle(newfolio);
|
| 809 |
|
| 810 | folio_migrate_refs(newfolio, folio);
|
| 811 | |
| 812 | * Copy NUMA information to the new page, to prevent over-eager
|
| 813 | * future migrations of this same page.
|
| 814 |
|
| 815 | cpupid = folio_xchg_last_cpupid(folio, -1);
|
| 816 | |
| 817 | * For memory tiering mode, when migrate between slow and fast
|
| 818 | * memory node, reset cpupid, because that is used to record
|
| 819 | * page access time in slow memory node.
|
| 820 |
|
| 821 | if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) {
|
| 822 | bool f_toptier = node_is_toptier(folio_nid(folio));
|
| 823 | bool t_toptier = node_is_toptier(folio_nid(newfolio));
|
| 824 |
|
| 825 | if (f_toptier != t_toptier)
|
| 826 | cpupid = -1;
|
| 827 | }
|
| 828 | folio_xchg_last_cpupid(newfolio, cpupid);
|
| 829 |
|
| 830 | folio_migrate_ksm(newfolio, folio);
|
| 831 | |
| 832 | * Please do not reorder this without considering how mm/ksm.c's
|
| 833 | * ksm_get_folio() depends upon ksm_migrate_page() and the
|
| 834 | * swapcache flag.
|
| 835 |
|
| 836 | if (folio_test_swapcache(folio))
|
| 837 | folio_clear_swapcache(folio);
|
| 838 | folio_clear_private(folio);
|
| 839 |
|
| 840 |
|
| 841 | if (!folio_test_hugetlb(folio))
|
| 842 | folio->private = NULL;
|
| 843 |
|
| 844 | |
| 845 | * If any waiters have accumulated on the new page then
|
| 846 | * wake them up.
|
| 847 |
|
| 848 | if (folio_test_writeback(newfolio))
|
| 849 | folio_end_writeback(newfolio);
|
| 850 |
|
| 851 | |
| 852 | * PG_readahead shares the same bit with PG_reclaim. The above
|
| 853 | * end_page_writeback() may clear PG_readahead mistakenly, so set the
|
| 854 | * bit after that.
|
| 855 |
|
| 856 | if (folio_test_readahead(folio))
|
| 857 | folio_set_readahead(newfolio);
|
| 858 |
|
| 859 | folio_copy_owner(newfolio, folio);
|
| 860 | pgalloc_tag_swap(newfolio, folio);
|
| 861 |
|
| 862 | mem_cgroup_migrate(folio, newfolio);
|
| 863 | }
|
| 864 | EXPORT_SYMBOL(folio_migrate_flags);
|
| 865 |
|
| 866 | |
| 867 | * Migration functions
|
| 868 | **********************************************************
|
| 869 |
|
| 870 | static int __migrate_folio(struct address_space *mapping, struct folio *dst,
|
| 871 | struct folio *src, void *src_private,
|
| 872 | enum migrate_mode mode)
|
| 873 | {
|
| 874 | int rc, expected_count = folio_expected_ref_count(src) + 1;
|
| 875 |
|
| 876 |
|
| 877 | if (folio_ref_count(src) != expected_count)
|
| 878 | return -EAGAIN;
|
| 879 |
|
| 880 | rc = folio_mc_copy(dst, src);
|
| 881 | if (unlikely(rc))
|
| 882 | return rc;
|
| 883 |
|
| 884 | rc = __folio_migrate_mapping(mapping, dst, src, expected_count);
|
| 885 | if (rc)
|
| 886 | return rc;
|
| 887 |
|
| 888 | if (src_private)
|
| 889 | folio_attach_private(dst, folio_detach_private(src));
|
| 890 |
|
| 891 | folio_migrate_flags(dst, src);
|
| 892 | return 0;
|
| 893 | }
|
| 894 |
|
| 895 | |
| 896 | * migrate_folio() - Simple folio migration.
|
| 897 | * @mapping: The address_space containing the folio.
|
| 898 | * @dst: The folio to migrate the data to.
|
| 899 | * @src: The folio containing the current data.
|
| 900 | * @mode: How to migrate the folio.
|
| 901 | *
|
| 902 | * Common logic to directly migrate a single LRU folio suitable for
|
| 903 | * folios that do not have private data.
|
| 904 | *
|
| 905 | * Folios are locked upon entry and exit.
|
| 906 |
|
| 907 | int migrate_folio(struct address_space *mapping, struct folio *dst,
|
| 908 | struct folio *src, enum migrate_mode mode)
|
| 909 | {
|
| 910 | BUG_ON(folio_test_writeback(src));
|
| 911 | return __migrate_folio(mapping, dst, src, NULL, mode);
|
| 912 | }
|
| 913 | EXPORT_SYMBOL(migrate_folio);
|
| 914 |
|
| 915 | #ifdef CONFIG_BUFFER_HEAD
|
| 916 |
|
| 917 | static bool buffer_migrate_lock_buffers(struct buffer_head *head,
|
| 918 | enum migrate_mode mode)
|
| 919 | {
|
| 920 | struct buffer_head *bh = head;
|
| 921 | struct buffer_head *failed_bh;
|
| 922 |
|
| 923 | do {
|
| 924 | if (!trylock_buffer(bh)) {
|
| 925 | if (mode == MIGRATE_ASYNC)
|
| 926 | goto unlock;
|
| 927 | if (mode == MIGRATE_SYNC_LIGHT && !buffer_uptodate(bh))
|
| 928 | goto unlock;
|
| 929 | lock_buffer(bh);
|
| 930 | }
|
| 931 |
|
| 932 | bh = bh->b_this_page;
|
| 933 | } while (bh != head);
|
| 934 |
|
| 935 | return true;
|
| 936 |
|
| 937 | unlock:
|
| 938 |
|
| 939 | failed_bh = bh;
|
| 940 | bh = head;
|
| 941 | while (bh != failed_bh) {
|
| 942 | unlock_buffer(bh);
|
| 943 | bh = bh->b_this_page;
|
| 944 | }
|
| 945 |
|
| 946 | return false;
|
| 947 | }
|
| 948 |
|
| 949 | static int __buffer_migrate_folio(struct address_space *mapping,
|
| 950 | struct folio *dst, struct folio *src, enum migrate_mode mode,
|
| 951 | bool check_refs)
|
| 952 | {
|
| 953 | struct buffer_head *bh, *head;
|
| 954 | int rc;
|
| 955 | int expected_count;
|
| 956 |
|
| 957 | head = folio_buffers(src);
|
| 958 | if (!head)
|
| 959 | return migrate_folio(mapping, dst, src, mode);
|
| 960 |
|
| 961 |
|
| 962 | expected_count = folio_expected_ref_count(src) + 1;
|
| 963 | if (folio_ref_count(src) != expected_count)
|
| 964 | return -EAGAIN;
|
| 965 |
|
| 966 | if (!buffer_migrate_lock_buffers(head, mode))
|
| 967 | return -EAGAIN;
|
| 968 |
|
| 969 | if (check_refs) {
|
| 970 | bool busy, migrating;
|
| 971 | bool invalidated = false;
|
| 972 |
|
| 973 | migrating = test_and_set_bit_lock(BH_Migrate, &head->b_state);
|
| 974 | VM_WARN_ON_ONCE(migrating);
|
| 975 | recheck_buffers:
|
| 976 | busy = false;
|
| 977 | spin_lock(&mapping->i_private_lock);
|
| 978 | bh = head;
|
| 979 | do {
|
| 980 | if (atomic_read(&bh->b_count)) {
|
| 981 | busy = true;
|
| 982 | break;
|
| 983 | }
|
| 984 | bh = bh->b_this_page;
|
| 985 | } while (bh != head);
|
| 986 | spin_unlock(&mapping->i_private_lock);
|
| 987 | if (busy) {
|
| 988 | if (invalidated) {
|
| 989 | rc = -EAGAIN;
|
| 990 | goto unlock_buffers;
|
| 991 | }
|
| 992 | invalidate_bh_lrus();
|
| 993 | invalidated = true;
|
| 994 | goto recheck_buffers;
|
| 995 | }
|
| 996 | }
|
| 997 |
|
| 998 | rc = filemap_migrate_folio(mapping, dst, src, mode);
|
| 999 | if (rc)
|
| 1000 | goto unlock_buffers;
|
| 1001 |
|
| 1002 | bh = head;
|
| 1003 | do {
|
| 1004 | folio_set_bh(bh, dst, bh_offset(bh));
|
| 1005 | bh = bh->b_this_page;
|
| 1006 | } while (bh != head);
|
| 1007 |
|
| 1008 | unlock_buffers:
|
| 1009 | if (check_refs)
|
| 1010 | clear_bit_unlock(BH_Migrate, &head->b_state);
|
| 1011 | bh = head;
|
| 1012 | do {
|
| 1013 | unlock_buffer(bh);
|
| 1014 | bh = bh->b_this_page;
|
| 1015 | } while (bh != head);
|
| 1016 |
|
| 1017 | return rc;
|
| 1018 | }
|
| 1019 |
|
| 1020 | |
| 1021 | * buffer_migrate_folio() - Migration function for folios with buffers.
|
| 1022 | * @mapping: The address space containing @src.
|
| 1023 | * @dst: The folio to migrate to.
|
| 1024 | * @src: The folio to migrate from.
|
| 1025 | * @mode: How to migrate the folio.
|
| 1026 | *
|
| 1027 | * This function can only be used if the underlying filesystem guarantees
|
| 1028 | * that no other references to @src exist. For example attached buffer
|
| 1029 | * heads are accessed only under the folio lock. If your filesystem cannot
|
| 1030 | * provide this guarantee, buffer_migrate_folio_norefs() may be more
|
| 1031 | * appropriate.
|
| 1032 | *
|
| 1033 | * Return: 0 on success or a negative errno on failure.
|
| 1034 |
|
| 1035 | int buffer_migrate_folio(struct address_space *mapping,
|
| 1036 | struct folio *dst, struct folio *src, enum migrate_mode mode)
|
| 1037 | {
|
| 1038 | return __buffer_migrate_folio(mapping, dst, src, mode, false);
|
| 1039 | }
|
| 1040 | EXPORT_SYMBOL(buffer_migrate_folio);
|
| 1041 |
|
| 1042 | |
| 1043 | * buffer_migrate_folio_norefs() - Migration function for folios with buffers.
|
| 1044 | * @mapping: The address space containing @src.
|
| 1045 | * @dst: The folio to migrate to.
|
| 1046 | * @src: The folio to migrate from.
|
| 1047 | * @mode: How to migrate the folio.
|
| 1048 | *
|
| 1049 | * Like buffer_migrate_folio() except that this variant is more careful
|
| 1050 | * and checks that there are also no buffer head references. This function
|
| 1051 | * is the right one for mappings where buffer heads are directly looked
|
| 1052 | * up and referenced (such as block device mappings).
|
| 1053 | *
|
| 1054 | * Return: 0 on success or a negative errno on failure.
|
| 1055 |
|
| 1056 | int buffer_migrate_folio_norefs(struct address_space *mapping,
|
| 1057 | struct folio *dst, struct folio *src, enum migrate_mode mode)
|
| 1058 | {
|
| 1059 | return __buffer_migrate_folio(mapping, dst, src, mode, true);
|
| 1060 | }
|
| 1061 | EXPORT_SYMBOL_GPL(buffer_migrate_folio_norefs);
|
| 1062 | #endif
|
| 1063 |
|
| 1064 | int filemap_migrate_folio(struct address_space *mapping,
|
| 1065 | struct folio *dst, struct folio *src, enum migrate_mode mode)
|
| 1066 | {
|
| 1067 | return __migrate_folio(mapping, dst, src, folio_get_private(src), mode);
|
| 1068 | }
|
| 1069 | EXPORT_SYMBOL_GPL(filemap_migrate_folio);
|
| 1070 |
|
| 1071 | |
| 1072 | * Default handling if a filesystem does not provide a migration function.
|
| 1073 |
|
| 1074 | static int fallback_migrate_folio(struct address_space *mapping,
|
| 1075 | struct folio *dst, struct folio *src, enum migrate_mode mode)
|
| 1076 | {
|
| 1077 | WARN_ONCE(mapping->a_ops->writepages,
|
| 1078 | "%ps does not implement migrate_folio\n",
|
| 1079 | mapping->a_ops);
|
| 1080 | if (folio_test_dirty(src))
|
| 1081 | return -EBUSY;
|
| 1082 |
|
| 1083 | |
| 1084 | * Filesystem may have private data at folio->private that we
|
| 1085 | * can't migrate automatically.
|
| 1086 |
|
| 1087 | if (!filemap_release_folio(src, GFP_KERNEL))
|
| 1088 | return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
|
| 1089 |
|
| 1090 | return migrate_folio(mapping, dst, src, mode);
|
| 1091 | }
|
| 1092 |
|
| 1093 | |
| 1094 | * Move a src folio to a newly allocated dst folio.
|
| 1095 | *
|
| 1096 | * The src and dst folios are locked and the src folios was unmapped from
|
| 1097 | * the page tables.
|
| 1098 | *
|
| 1099 | * On success, the src folio was replaced by the dst folio.
|
| 1100 | *
|
| 1101 | * Return value:
|
| 1102 | * < 0 - error code
|
| 1103 | * 0 - success
|
| 1104 |
|
| 1105 | static int move_to_new_folio(struct folio *dst, struct folio *src,
|
| 1106 | enum migrate_mode mode)
|
| 1107 | {
|
| 1108 | struct address_space *mapping = folio_mapping(src);
|
| 1109 | int rc = -EAGAIN;
|
| 1110 |
|
| 1111 | VM_BUG_ON_FOLIO(!folio_test_locked(src), src);
|
| 1112 | VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst);
|
| 1113 |
|
| 1114 | if (!mapping)
|
| 1115 | rc = migrate_folio(mapping, dst, src, mode);
|
| 1116 | else if (mapping_inaccessible(mapping))
|
| 1117 | rc = -EOPNOTSUPP;
|
| 1118 | else if (mapping->a_ops->migrate_folio)
|
| 1119 | |
| 1120 | * Most folios have a mapping and most filesystems
|
| 1121 | * provide a migrate_folio callback. Anonymous folios
|
| 1122 | * are part of swap space which also has its own
|
| 1123 | * migrate_folio callback. This is the most common path
|
| 1124 | * for page migration.
|
| 1125 |
|
| 1126 | rc = mapping->a_ops->migrate_folio(mapping, dst, src,
|
| 1127 | mode);
|
| 1128 | else
|
| 1129 | rc = fallback_migrate_folio(mapping, dst, src, mode);
|
| 1130 |
|
| 1131 | if (!rc) {
|
| 1132 | |
| 1133 | * For pagecache folios, src->mapping must be cleared before src
|
| 1134 | * is freed. Anonymous folios must stay anonymous until freed.
|
| 1135 |
|
| 1136 | if (!folio_test_anon(src))
|
| 1137 | src->mapping = NULL;
|
| 1138 |
|
| 1139 | if (likely(!folio_is_zone_device(dst)))
|
| 1140 | flush_dcache_folio(dst);
|
| 1141 | }
|
| 1142 | return rc;
|
| 1143 | }
|
| 1144 |
|
| 1145 | |
| 1146 | * To record some information during migration, we use the migrate_info
|
| 1147 | * field of struct folio of the newly allocated destination folio.
|
| 1148 | * This is safe because nobody is using it except us.
|
| 1149 |
|
| 1150 | enum {
|
| 1151 | FOLIO_WAS_MAPPED = BIT(0),
|
| 1152 | FOLIO_WAS_MLOCKED = BIT(1),
|
| 1153 | FOLIO_OLD_STATES = FOLIO_WAS_MAPPED | FOLIO_WAS_MLOCKED,
|
| 1154 | };
|
| 1155 |
|
| 1156 | static void __migrate_folio_record(struct folio *dst,
|
| 1157 | int old_folio_state, struct anon_vma *anon_vma)
|
| 1158 | {
|
| 1159 | dst->migrate_info = (unsigned long)anon_vma | old_folio_state;
|
| 1160 | }
|
| 1161 |
|
| 1162 | static void __migrate_folio_extract(struct folio *dst,
|
| 1163 | int *old_folio_state, struct anon_vma **anon_vmap)
|
| 1164 | {
|
| 1165 | unsigned long info = dst->migrate_info;
|
| 1166 |
|
| 1167 | *anon_vmap = (struct anon_vma *)(info & ~FOLIO_OLD_STATES);
|
| 1168 | *old_folio_state = info & FOLIO_OLD_STATES;
|
| 1169 | dst->migrate_info = 0;
|
| 1170 | }
|
| 1171 |
|
| 1172 |
|
| 1173 | static void migrate_folio_undo_src(struct folio *src, int was_mapped,
|
| 1174 | struct anon_vma *anon_vma, bool locked, struct list_head *ret)
|
| 1175 | {
|
| 1176 | if (was_mapped)
|
| 1177 | remove_migration_ptes(src, src, 0);
|
| 1178 |
|
| 1179 | if (anon_vma)
|
| 1180 | put_anon_vma(anon_vma);
|
| 1181 | if (locked)
|
| 1182 | folio_unlock(src);
|
| 1183 | if (ret)
|
| 1184 | list_move_tail(&src->lru, ret);
|
| 1185 | }
|
| 1186 |
|
| 1187 |
|
| 1188 | static void migrate_folio_undo_dst(struct folio *dst, bool locked,
|
| 1189 | free_folio_t put_new_folio, unsigned long private)
|
| 1190 | {
|
| 1191 | if (locked)
|
| 1192 | folio_unlock(dst);
|
| 1193 | if (put_new_folio)
|
| 1194 | put_new_folio(dst, private);
|
| 1195 | else
|
| 1196 | folio_put(dst);
|
| 1197 | }
|
| 1198 |
|
| 1199 |
|
| 1200 | static void migrate_folio_done(struct folio *src,
|
| 1201 | enum migrate_reason reason)
|
| 1202 | {
|
| 1203 | if (likely(!page_has_movable_ops(&src->page)) && reason != MR_DEMOTION)
|
| 1204 | mod_node_page_state(folio_pgdat(src), NR_ISOLATED_ANON +
|
| 1205 | folio_is_file_lru(src), -folio_nr_pages(src));
|
| 1206 |
|
| 1207 | if (reason != MR_MEMORY_FAILURE)
|
| 1208 |
|
| 1209 | folio_put(src);
|
| 1210 | }
|
| 1211 |
|
| 1212 |
|
| 1213 | static int migrate_folio_unmap(new_folio_t get_new_folio,
|
| 1214 | free_folio_t put_new_folio, unsigned long private,
|
| 1215 | struct folio *src, struct folio **dstp, enum migrate_mode mode,
|
| 1216 | struct list_head *ret)
|
| 1217 | {
|
| 1218 | struct folio *dst;
|
| 1219 | int rc = -EAGAIN;
|
| 1220 | int old_folio_state = 0;
|
| 1221 | struct anon_vma *anon_vma = NULL;
|
| 1222 | bool locked = false;
|
| 1223 | bool dst_locked = false;
|
| 1224 |
|
| 1225 | dst = get_new_folio(src, private);
|
| 1226 | if (!dst)
|
| 1227 | return -ENOMEM;
|
| 1228 | *dstp = dst;
|
| 1229 |
|
| 1230 | dst->migrate_info = 0;
|
| 1231 |
|
| 1232 | if (!folio_trylock(src)) {
|
| 1233 | if (mode == MIGRATE_ASYNC)
|
| 1234 | goto out;
|
| 1235 |
|
| 1236 | |
| 1237 | * It's not safe for direct compaction to call lock_page.
|
| 1238 | * For example, during page readahead pages are added locked
|
| 1239 | * to the LRU. Later, when the IO completes the pages are
|
| 1240 | * marked uptodate and unlocked. However, the queueing
|
| 1241 | * could be merging multiple pages for one bio (e.g.
|
| 1242 | * mpage_readahead). If an allocation happens for the
|
| 1243 | * second or third page, the process can end up locking
|
| 1244 | * the same page twice and deadlocking. Rather than
|
| 1245 | * trying to be clever about what pages can be locked,
|
| 1246 | * avoid the use of lock_page for direct compaction
|
| 1247 | * altogether.
|
| 1248 |
|
| 1249 | if (current->flags & PF_MEMALLOC)
|
| 1250 | goto out;
|
| 1251 |
|
| 1252 | |
| 1253 | * In "light" mode, we can wait for transient locks (eg
|
| 1254 | * inserting a page into the page table), but it's not
|
| 1255 | * worth waiting for I/O.
|
| 1256 |
|
| 1257 | if (mode == MIGRATE_SYNC_LIGHT && !folio_test_uptodate(src))
|
| 1258 | goto out;
|
| 1259 |
|
| 1260 | folio_lock(src);
|
| 1261 | }
|
| 1262 | locked = true;
|
| 1263 | if (folio_test_mlocked(src))
|
| 1264 | old_folio_state |= FOLIO_WAS_MLOCKED;
|
| 1265 |
|
| 1266 | if (folio_test_writeback(src)) {
|
| 1267 | |
| 1268 | * Only in the case of a full synchronous migration is it
|
| 1269 | * necessary to wait for writeback. In the async case,
|
| 1270 | * the retry loop is too short and in the sync-light case,
|
| 1271 | * the overhead of stalling is too much
|
| 1272 |
|
| 1273 | switch (mode) {
|
| 1274 | case MIGRATE_SYNC:
|
| 1275 | break;
|
| 1276 | default:
|
| 1277 | rc = -EBUSY;
|
| 1278 | goto out;
|
| 1279 | }
|
| 1280 | folio_wait_writeback(src);
|
| 1281 | }
|
| 1282 |
|
| 1283 | |
| 1284 | * By try_to_migrate(), src->mapcount goes down to 0 here. In this case,
|
| 1285 | * we cannot notice that anon_vma is freed while we migrate a page.
|
| 1286 | * This get_anon_vma() delays freeing anon_vma pointer until the end
|
| 1287 | * of migration. File cache pages are no problem because of page_lock()
|
| 1288 | * File Caches may use write_page() or lock_page() in migration, then,
|
| 1289 | * just care Anon page here.
|
| 1290 | *
|
| 1291 | * Only folio_get_anon_vma() understands the subtleties of
|
| 1292 | * getting a hold on an anon_vma from outside one of its mms.
|
| 1293 | * But if we cannot get anon_vma, then we won't need it anyway,
|
| 1294 | * because that implies that the anon page is no longer mapped
|
| 1295 | * (and cannot be remapped so long as we hold the page lock).
|
| 1296 |
|
| 1297 | if (folio_test_anon(src) && !folio_test_ksm(src))
|
| 1298 | anon_vma = folio_get_anon_vma(src);
|
| 1299 |
|
| 1300 | |
| 1301 | * Block others from accessing the new page when we get around to
|
| 1302 | * establishing additional references. We are usually the only one
|
| 1303 | * holding a reference to dst at this point. We used to have a BUG
|
| 1304 | * here if folio_trylock(dst) fails, but would like to allow for
|
| 1305 | * cases where there might be a race with the previous use of dst.
|
| 1306 | * This is much like races on refcount of oldpage: just don't BUG().
|
| 1307 |
|
| 1308 | if (unlikely(!folio_trylock(dst)))
|
| 1309 | goto out;
|
| 1310 | dst_locked = true;
|
| 1311 |
|
| 1312 | if (unlikely(page_has_movable_ops(&src->page))) {
|
| 1313 | __migrate_folio_record(dst, old_folio_state, anon_vma);
|
| 1314 | return 0;
|
| 1315 | }
|
| 1316 |
|
| 1317 | |
| 1318 | * Corner case handling:
|
| 1319 | * 1. When a new swap-cache page is read into, it is added to the LRU
|
| 1320 | * and treated as swapcache but it has no rmap yet.
|
| 1321 | * Calling try_to_unmap() against a src->mapping==NULL page will
|
| 1322 | * trigger a BUG. So handle it here.
|
| 1323 | * 2. An orphaned page (see truncate_cleanup_page) might have
|
| 1324 | * fs-private metadata. The page can be picked up due to memory
|
| 1325 | * offlining. Everywhere else except page reclaim, the page is
|
| 1326 | * invisible to the vm, so the page can not be migrated. So try to
|
| 1327 | * free the metadata, so the page can be freed.
|
| 1328 |
|
| 1329 | if (!src->mapping) {
|
| 1330 | if (folio_test_private(src)) {
|
| 1331 | try_to_free_buffers(src);
|
| 1332 | goto out;
|
| 1333 | }
|
| 1334 | } else if (folio_mapped(src)) {
|
| 1335 |
|
| 1336 | VM_BUG_ON_FOLIO(folio_test_anon(src) &&
|
| 1337 | !folio_test_ksm(src) && !anon_vma, src);
|
| 1338 | try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
|
| 1339 | old_folio_state |= FOLIO_WAS_MAPPED;
|
| 1340 | }
|
| 1341 |
|
| 1342 | if (!folio_mapped(src)) {
|
| 1343 | __migrate_folio_record(dst, old_folio_state, anon_vma);
|
| 1344 | return 0;
|
| 1345 | }
|
| 1346 |
|
| 1347 | out:
|
| 1348 | |
| 1349 | * A folio that has not been unmapped will be restored to
|
| 1350 | * right list unless we want to retry.
|
| 1351 |
|
| 1352 | if (rc == -EAGAIN)
|
| 1353 | ret = NULL;
|
| 1354 |
|
| 1355 | migrate_folio_undo_src(src, old_folio_state & FOLIO_WAS_MAPPED,
|
| 1356 | anon_vma, locked, ret);
|
| 1357 | migrate_folio_undo_dst(dst, dst_locked, put_new_folio, private);
|
| 1358 |
|
| 1359 | return rc;
|
| 1360 | }
|
| 1361 |
|
| 1362 |
|
| 1363 | static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
|
| 1364 | struct folio *src, struct folio *dst,
|
| 1365 | enum migrate_mode mode, enum migrate_reason reason,
|
| 1366 | struct list_head *ret)
|
| 1367 | {
|
| 1368 | int rc;
|
| 1369 | int old_folio_state = 0;
|
| 1370 | struct anon_vma *anon_vma = NULL;
|
| 1371 | bool src_deferred_split = false;
|
| 1372 | bool src_partially_mapped = false;
|
| 1373 | struct list_head *prev;
|
| 1374 |
|
| 1375 | __migrate_folio_extract(dst, &old_folio_state, &anon_vma);
|
| 1376 | prev = dst->lru.prev;
|
| 1377 | list_del(&dst->lru);
|
| 1378 |
|
| 1379 | if (unlikely(page_has_movable_ops(&src->page))) {
|
| 1380 | rc = migrate_movable_ops_page(&dst->page, &src->page, mode);
|
| 1381 | if (rc)
|
| 1382 | goto out;
|
| 1383 | goto out_unlock_both;
|
| 1384 | }
|
| 1385 |
|
| 1386 | if (folio_order(src) > 1 &&
|
| 1387 | !data_race(list_empty(&src->_deferred_list))) {
|
| 1388 | src_deferred_split = true;
|
| 1389 | src_partially_mapped = folio_test_partially_mapped(src);
|
| 1390 | }
|
| 1391 |
|
| 1392 | rc = move_to_new_folio(dst, src, mode);
|
| 1393 | if (rc)
|
| 1394 | goto out;
|
| 1395 |
|
| 1396 | |
| 1397 | * Requeue the destination folio on the deferred split queue if
|
| 1398 | * the source was on the queue. The source is unqueued in
|
| 1399 | * __folio_migrate_mapping(), so we recorded the state from
|
| 1400 | * before move_to_new_folio().
|
| 1401 |
|
| 1402 | if (src_deferred_split)
|
| 1403 | deferred_split_folio(dst, src_partially_mapped);
|
| 1404 |
|
| 1405 | |
| 1406 | * When successful, push dst to LRU immediately: so that if it
|
| 1407 | * turns out to be an mlocked page, remove_migration_ptes() will
|
| 1408 | * automatically build up the correct dst->mlock_count for it.
|
| 1409 | *
|
| 1410 | * We would like to do something similar for the old page, when
|
| 1411 | * unsuccessful, and other cases when a page has been temporarily
|
| 1412 | * isolated from the unevictable LRU: but this case is the easiest.
|
| 1413 |
|
| 1414 | folio_add_lru(dst);
|
| 1415 | if (old_folio_state & FOLIO_WAS_MLOCKED)
|
| 1416 | lru_add_drain();
|
| 1417 |
|
| 1418 | if (old_folio_state & FOLIO_WAS_MAPPED)
|
| 1419 | remove_migration_ptes(src, dst, 0);
|
| 1420 |
|
| 1421 | out_unlock_both:
|
| 1422 | folio_unlock(dst);
|
| 1423 | folio_set_owner_migrate_reason(dst, reason);
|
| 1424 | |
| 1425 | * If migration is successful, decrease refcount of dst,
|
| 1426 | * which will not free the page because new page owner increased
|
| 1427 | * refcounter.
|
| 1428 |
|
| 1429 | folio_put(dst);
|
| 1430 |
|
| 1431 | |
| 1432 | * A folio that has been migrated has all references removed
|
| 1433 | * and will be freed.
|
| 1434 |
|
| 1435 | list_del(&src->lru);
|
| 1436 |
|
| 1437 | if (anon_vma)
|
| 1438 | put_anon_vma(anon_vma);
|
| 1439 | folio_unlock(src);
|
| 1440 | migrate_folio_done(src, reason);
|
| 1441 |
|
| 1442 | return rc;
|
| 1443 | out:
|
| 1444 | |
| 1445 | * A folio that has not been migrated will be restored to
|
| 1446 | * right list unless we want to retry.
|
| 1447 |
|
| 1448 | if (rc == -EAGAIN) {
|
| 1449 | list_add(&dst->lru, prev);
|
| 1450 | __migrate_folio_record(dst, old_folio_state, anon_vma);
|
| 1451 | return rc;
|
| 1452 | }
|
| 1453 |
|
| 1454 | migrate_folio_undo_src(src, old_folio_state & FOLIO_WAS_MAPPED,
|
| 1455 | anon_vma, true, ret);
|
| 1456 | migrate_folio_undo_dst(dst, true, put_new_folio, private);
|
| 1457 |
|
| 1458 | return rc;
|
| 1459 | }
|
| 1460 |
|
| 1461 | |
| 1462 | * Counterpart of migrate_folio_unmap() and migrate_folio_move() for hugetlb
|
| 1463 | * folio migration.
|
| 1464 | *
|
| 1465 | * This function doesn't wait the completion of hugepage I/O
|
| 1466 | * because there is no race between I/O and migration for hugepage.
|
| 1467 | * Note that currently hugepage I/O occurs only in direct I/O
|
| 1468 | * where no lock is held and PG_writeback is irrelevant,
|
| 1469 | * and writeback status of all subpages are counted in the reference
|
| 1470 | * count of the head page (i.e. if all subpages of a 2MB hugepage are
|
| 1471 | * under direct I/O, the reference of the head page is 512 and a bit more.)
|
| 1472 | * This means that when we try to migrate hugepage whose subpages are
|
| 1473 | * doing direct I/O, some references remain after try_to_unmap() and
|
| 1474 | * hugepage migration fails without data corruption.
|
| 1475 | *
|
| 1476 | * There is also no race when direct I/O is issued on the page under migration,
|
| 1477 | * because then pte is replaced with migration swap entry and direct I/O code
|
| 1478 | * will wait in the page fault for migration to complete.
|
| 1479 |
|
| 1480 | static int unmap_and_move_hugetlb_folio(new_folio_t get_new_folio,
|
| 1481 | free_folio_t put_new_folio, unsigned long private,
|
| 1482 | struct folio *src, int force, enum migrate_mode mode,
|
| 1483 | enum migrate_reason reason, struct list_head *ret)
|
| 1484 | {
|
| 1485 | struct folio *dst;
|
| 1486 | int rc = -EAGAIN;
|
| 1487 | int was_mapped = 0;
|
| 1488 | struct anon_vma *anon_vma = NULL;
|
| 1489 | struct address_space *mapping = NULL;
|
| 1490 | enum ttu_flags ttu = 0;
|
| 1491 |
|
| 1492 | if (folio_ref_count(src) == 1) {
|
| 1493 |
|
| 1494 | folio_putback_hugetlb(src);
|
| 1495 | return 0;
|
| 1496 | }
|
| 1497 |
|
| 1498 | dst = get_new_folio(src, private);
|
| 1499 | if (!dst)
|
| 1500 | return -ENOMEM;
|
| 1501 |
|
| 1502 | if (!folio_trylock(src)) {
|
| 1503 | if (!force)
|
| 1504 | goto out;
|
| 1505 | switch (mode) {
|
| 1506 | case MIGRATE_SYNC:
|
| 1507 | break;
|
| 1508 | default:
|
| 1509 | goto out;
|
| 1510 | }
|
| 1511 | folio_lock(src);
|
| 1512 | }
|
| 1513 |
|
| 1514 | |
| 1515 | * Check for folios which are in the process of being freed. Without
|
| 1516 | * folio_mapping() set, hugetlbfs specific move folio routine will not
|
| 1517 | * be called and we could leak usage counts for subpools.
|
| 1518 |
|
| 1519 | if (hugetlb_folio_subpool(src) && !folio_mapping(src)) {
|
| 1520 | rc = -EBUSY;
|
| 1521 | goto out_unlock;
|
| 1522 | }
|
| 1523 |
|
| 1524 | if (folio_test_anon(src))
|
| 1525 | anon_vma = folio_get_anon_vma(src);
|
| 1526 |
|
| 1527 | if (unlikely(!folio_trylock(dst)))
|
| 1528 | goto put_anon;
|
| 1529 |
|
| 1530 | if (folio_mapped(src)) {
|
| 1531 | if (!folio_test_anon(src)) {
|
| 1532 | |
| 1533 | * In shared mappings, try_to_unmap could potentially
|
| 1534 | * call huge_pmd_unshare. Because of this, take
|
| 1535 | * semaphore in write mode here and set TTU_RMAP_LOCKED
|
| 1536 | * to let lower levels know we have taken the lock.
|
| 1537 |
|
| 1538 | mapping = hugetlb_folio_mapping_lock_write(src);
|
| 1539 | if (unlikely(!mapping))
|
| 1540 | goto unlock_put_anon;
|
| 1541 |
|
| 1542 | ttu = TTU_RMAP_LOCKED;
|
| 1543 | }
|
| 1544 |
|
| 1545 | try_to_migrate(src, ttu);
|
| 1546 | was_mapped = 1;
|
| 1547 | }
|
| 1548 |
|
| 1549 | if (!folio_mapped(src))
|
| 1550 | rc = move_to_new_folio(dst, src, mode);
|
| 1551 |
|
| 1552 | if (was_mapped)
|
| 1553 | remove_migration_ptes(src, !rc ? dst : src, ttu);
|
| 1554 |
|
| 1555 | if (ttu & TTU_RMAP_LOCKED)
|
| 1556 | i_mmap_unlock_write(mapping);
|
| 1557 |
|
| 1558 | unlock_put_anon:
|
| 1559 | folio_unlock(dst);
|
| 1560 |
|
| 1561 | put_anon:
|
| 1562 | if (anon_vma)
|
| 1563 | put_anon_vma(anon_vma);
|
| 1564 |
|
| 1565 | if (!rc) {
|
| 1566 | move_hugetlb_state(src, dst, reason);
|
| 1567 | put_new_folio = NULL;
|
| 1568 | }
|
| 1569 |
|
| 1570 | out_unlock:
|
| 1571 | folio_unlock(src);
|
| 1572 | out:
|
| 1573 | if (!rc)
|
| 1574 | folio_putback_hugetlb(src);
|
| 1575 | else if (rc != -EAGAIN)
|
| 1576 | list_move_tail(&src->lru, ret);
|
| 1577 |
|
| 1578 | |
| 1579 | * If migration was not successful and there's a freeing callback,
|
| 1580 | * return the folio to that special allocator. Otherwise, simply drop
|
| 1581 | * our additional reference.
|
| 1582 |
|
| 1583 | if (put_new_folio)
|
| 1584 | put_new_folio(dst, private);
|
| 1585 | else
|
| 1586 | folio_put(dst);
|
| 1587 |
|
| 1588 | return rc;
|
| 1589 | }
|
| 1590 |
|
| 1591 | static inline int try_split_folio(struct folio *folio, struct list_head *split_folios,
|
| 1592 | enum migrate_mode mode)
|
| 1593 | {
|
| 1594 | int rc;
|
| 1595 |
|
| 1596 | if (mode == MIGRATE_ASYNC) {
|
| 1597 | if (!folio_trylock(folio))
|
| 1598 | return -EAGAIN;
|
| 1599 | } else {
|
| 1600 | folio_lock(folio);
|
| 1601 | }
|
| 1602 | rc = split_folio_to_list(folio, split_folios);
|
| 1603 | folio_unlock(folio);
|
| 1604 | if (!rc)
|
| 1605 | list_move_tail(&folio->lru, split_folios);
|
| 1606 |
|
| 1607 | return rc;
|
| 1608 | }
|
| 1609 |
|
| 1610 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
| 1611 | #define NR_MAX_BATCHED_MIGRATION HPAGE_PMD_NR
|
| 1612 | #else
|
| 1613 | #define NR_MAX_BATCHED_MIGRATION 512
|
| 1614 | #endif
|
| 1615 | #define NR_MAX_MIGRATE_PAGES_RETRY 10
|
| 1616 | #define NR_MAX_MIGRATE_ASYNC_RETRY 3
|
| 1617 | #define NR_MAX_MIGRATE_SYNC_RETRY \
|
| 1618 | (NR_MAX_MIGRATE_PAGES_RETRY - NR_MAX_MIGRATE_ASYNC_RETRY)
|
| 1619 |
|
| 1620 | struct migrate_pages_stats {
|
| 1621 | int nr_succeeded; |
| 1622 | units of base pages
|
| 1623 | int nr_failed_pages; |
| 1624 | units of base pages. Untried folios aren't counted
|
| 1625 | int nr_thp_succeeded;
|
| 1626 | int nr_thp_failed;
|
| 1627 | int nr_thp_split;
|
| 1628 | int nr_split;
|
| 1629 | };
|
| 1630 |
|
| 1631 | |
| 1632 | * Returns the number of hugetlb folios that were not migrated, or an error code
|
| 1633 | * after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no hugetlb folios are movable
|
| 1634 | * any more because the list has become empty or no retryable hugetlb folios
|
| 1635 | * exist any more. It is caller's responsibility to call putback_movable_pages()
|
| 1636 | * only if ret != 0.
|
| 1637 |
|
| 1638 | static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio,
|
| 1639 | free_folio_t put_new_folio, unsigned long private,
|
| 1640 | enum migrate_mode mode, enum migrate_reason reason,
|
| 1641 | struct migrate_pages_stats *stats,
|
| 1642 | struct list_head *ret_folios)
|
| 1643 | {
|
| 1644 | int retry = 1;
|
| 1645 | int nr_failed = 0;
|
| 1646 | int nr_retry_pages = 0;
|
| 1647 | int pass = 0;
|
| 1648 | struct folio *folio, *folio2;
|
| 1649 | int rc, nr_pages;
|
| 1650 |
|
| 1651 | for (pass = 0; pass < NR_MAX_MIGRATE_PAGES_RETRY && retry; pass++) {
|
| 1652 | retry = 0;
|
| 1653 | nr_retry_pages = 0;
|
| 1654 |
|
| 1655 | list_for_each_entry_safe(folio, folio2, from, lru) {
|
| 1656 | if (!folio_test_hugetlb(folio))
|
| 1657 | continue;
|
| 1658 |
|
| 1659 | nr_pages = folio_nr_pages(folio);
|
| 1660 |
|
| 1661 | cond_resched();
|
| 1662 |
|
| 1663 | |
| 1664 | * Migratability of hugepages depends on architectures and
|
| 1665 | * their size. This check is necessary because some callers
|
| 1666 | * of hugepage migration like soft offline and memory
|
| 1667 | * hotremove don't walk through page tables or check whether
|
| 1668 | * the hugepage is pmd-based or not before kicking migration.
|
| 1669 |
|
| 1670 | if (!hugepage_migration_supported(folio_hstate(folio))) {
|
| 1671 | nr_failed++;
|
| 1672 | stats->nr_failed_pages += nr_pages;
|
| 1673 | list_move_tail(&folio->lru, ret_folios);
|
| 1674 | continue;
|
| 1675 | }
|
| 1676 |
|
| 1677 | rc = unmap_and_move_hugetlb_folio(get_new_folio,
|
| 1678 | put_new_folio, private,
|
| 1679 | folio, pass > 2, mode,
|
| 1680 | reason, ret_folios);
|
| 1681 | |
| 1682 | * The rules are:
|
| 1683 | * 0: hugetlb folio will be put back
|
| 1684 | * -EAGAIN: stay on the from list
|
| 1685 | * -ENOMEM: stay on the from list
|
| 1686 | * Other errno: put on ret_folios list
|
| 1687 |
|
| 1688 | switch(rc) {
|
| 1689 | case -ENOMEM:
|
| 1690 | |
| 1691 | * When memory is low, don't bother to try to migrate
|
| 1692 | * other folios, just exit.
|
| 1693 |
|
| 1694 | stats->nr_failed_pages += nr_pages + nr_retry_pages;
|
| 1695 | return -ENOMEM;
|
| 1696 | case -EAGAIN:
|
| 1697 | retry++;
|
| 1698 | nr_retry_pages += nr_pages;
|
| 1699 | break;
|
| 1700 | case 0:
|
| 1701 | stats->nr_succeeded += nr_pages;
|
| 1702 | break;
|
| 1703 | default:
|
| 1704 | |
| 1705 | * Permanent failure (-EBUSY, etc.):
|
| 1706 | * unlike -EAGAIN case, the failed folio is
|
| 1707 | * removed from migration folio list and not
|
| 1708 | * retried in the next outer loop.
|
| 1709 |
|
| 1710 | nr_failed++;
|
| 1711 | stats->nr_failed_pages += nr_pages;
|
| 1712 | break;
|
| 1713 | }
|
| 1714 | }
|
| 1715 | }
|
| 1716 | |
| 1717 | * nr_failed is number of hugetlb folios failed to be migrated. After
|
| 1718 | * NR_MAX_MIGRATE_PAGES_RETRY attempts, give up and count retried hugetlb
|
| 1719 | * folios as failed.
|
| 1720 |
|
| 1721 | nr_failed += retry;
|
| 1722 | stats->nr_failed_pages += nr_retry_pages;
|
| 1723 |
|
| 1724 | return nr_failed;
|
| 1725 | }
|
| 1726 |
|
| 1727 | static void migrate_folios_move(struct list_head *src_folios,
|
| 1728 | struct list_head *dst_folios,
|
| 1729 | free_folio_t put_new_folio, unsigned long private,
|
| 1730 | enum migrate_mode mode, enum migrate_reason reason,
|
| 1731 | struct list_head *ret_folios,
|
| 1732 | struct migrate_pages_stats *stats,
|
| 1733 | int *retry, int *thp_retry, int *nr_failed,
|
| 1734 | int *nr_retry_pages)
|
| 1735 | {
|
| 1736 | struct folio *folio, *folio2, *dst, *dst2;
|
| 1737 | bool is_thp;
|
| 1738 | int nr_pages;
|
| 1739 | int rc;
|
| 1740 |
|
| 1741 | dst = list_first_entry(dst_folios, struct folio, lru);
|
| 1742 | dst2 = list_next_entry(dst, lru);
|
| 1743 | list_for_each_entry_safe(folio, folio2, src_folios, lru) {
|
| 1744 | is_thp = folio_test_large(folio) && folio_test_pmd_mappable(folio);
|
| 1745 | nr_pages = folio_nr_pages(folio);
|
| 1746 |
|
| 1747 | cond_resched();
|
| 1748 |
|
| 1749 | rc = migrate_folio_move(put_new_folio, private,
|
| 1750 | folio, dst, mode,
|
| 1751 | reason, ret_folios);
|
| 1752 | |
| 1753 | * The rules are:
|
| 1754 | * 0: folio will be freed
|
| 1755 | * -EAGAIN: stay on the src_folios list
|
| 1756 | * Other errno: put on ret_folios list
|
| 1757 |
|
| 1758 | switch (rc) {
|
| 1759 | case -EAGAIN:
|
| 1760 | *retry += 1;
|
| 1761 | *thp_retry += is_thp;
|
| 1762 | *nr_retry_pages += nr_pages;
|
| 1763 | break;
|
| 1764 | case 0:
|
| 1765 | stats->nr_succeeded += nr_pages;
|
| 1766 | stats->nr_thp_succeeded += is_thp;
|
| 1767 | break;
|
| 1768 | default:
|
| 1769 | *nr_failed += 1;
|
| 1770 | stats->nr_thp_failed += is_thp;
|
| 1771 | stats->nr_failed_pages += nr_pages;
|
| 1772 | break;
|
| 1773 | }
|
| 1774 | dst = dst2;
|
| 1775 | dst2 = list_next_entry(dst, lru);
|
| 1776 | }
|
| 1777 | }
|
| 1778 |
|
| 1779 | static void migrate_folios_undo(struct list_head *src_folios,
|
| 1780 | struct list_head *dst_folios,
|
| 1781 | free_folio_t put_new_folio, unsigned long private,
|
| 1782 | struct list_head *ret_folios)
|
| 1783 | {
|
| 1784 | struct folio *folio, *folio2, *dst, *dst2;
|
| 1785 |
|
| 1786 | dst = list_first_entry(dst_folios, struct folio, lru);
|
| 1787 | dst2 = list_next_entry(dst, lru);
|
| 1788 | list_for_each_entry_safe(folio, folio2, src_folios, lru) {
|
| 1789 | int old_folio_state = 0;
|
| 1790 | struct anon_vma *anon_vma = NULL;
|
| 1791 |
|
| 1792 | __migrate_folio_extract(dst, &old_folio_state, &anon_vma);
|
| 1793 | migrate_folio_undo_src(folio, old_folio_state & FOLIO_WAS_MAPPED,
|
| 1794 | anon_vma, true, ret_folios);
|
| 1795 | list_del(&dst->lru);
|
| 1796 | migrate_folio_undo_dst(dst, true, put_new_folio, private);
|
| 1797 | dst = dst2;
|
| 1798 | dst2 = list_next_entry(dst, lru);
|
| 1799 | }
|
| 1800 | }
|
| 1801 |
|
| 1802 | |
| 1803 | * migrate_pages_batch() first unmaps folios in the from list as many as
|
| 1804 | * possible, then move the unmapped folios.
|
| 1805 | *
|
| 1806 | * We only batch migration if mode == MIGRATE_ASYNC to avoid to wait a
|
| 1807 | * lock or bit when we have locked more than one folio. Which may cause
|
| 1808 | * deadlock (e.g., for loop device). So, if mode != MIGRATE_ASYNC, the
|
| 1809 | * length of the from list must be <= 1.
|
| 1810 |
|
| 1811 | static int migrate_pages_batch(struct list_head *from,
|
| 1812 | new_folio_t get_new_folio, free_folio_t put_new_folio,
|
| 1813 | unsigned long private, enum migrate_mode mode, enum migrate_reason reason,
|
| 1814 | struct list_head *ret_folios, struct list_head *split_folios,
|
| 1815 | struct migrate_pages_stats *stats, int nr_pass)
|
| 1816 | {
|
| 1817 | int retry = 1;
|
| 1818 | int thp_retry = 1;
|
| 1819 | int nr_failed = 0;
|
| 1820 | int nr_retry_pages = 0;
|
| 1821 | int pass = 0;
|
| 1822 | bool is_thp = false;
|
| 1823 | bool is_large = false;
|
| 1824 | struct folio *folio, *folio2, *dst = NULL;
|
| 1825 | int rc, rc_saved = 0, nr_pages;
|
| 1826 | LIST_HEAD(unmap_folios);
|
| 1827 | LIST_HEAD(dst_folios);
|
| 1828 | bool nosplit = (reason == MR_NUMA_MISPLACED);
|
| 1829 |
|
| 1830 | VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC &&
|
| 1831 | !list_empty(from) && !list_is_singular(from));
|
| 1832 |
|
| 1833 | for (pass = 0; pass < nr_pass && retry; pass++) {
|
| 1834 | retry = 0;
|
| 1835 | thp_retry = 0;
|
| 1836 | nr_retry_pages = 0;
|
| 1837 |
|
| 1838 | list_for_each_entry_safe(folio, folio2, from, lru) {
|
| 1839 | is_large = folio_test_large(folio);
|
| 1840 | is_thp = folio_test_pmd_mappable(folio);
|
| 1841 | nr_pages = folio_nr_pages(folio);
|
| 1842 |
|
| 1843 | cond_resched_tasks_rcu_qs();
|
| 1844 |
|
| 1845 | |
| 1846 | * The rare folio on the deferred split list should
|
| 1847 | * be split now. It should not count as a failure:
|
| 1848 | * but increment nr_failed because, without doing so,
|
| 1849 | * migrate_pages() may report success with (split but
|
| 1850 | * unmigrated) pages still on its fromlist; whereas it
|
| 1851 | * always reports success when its fromlist is empty.
|
| 1852 | * stats->nr_thp_failed should be increased too,
|
| 1853 | * otherwise stats inconsistency will happen when
|
| 1854 | * migrate_pages_batch is called via migrate_pages()
|
| 1855 | * with MIGRATE_SYNC and MIGRATE_ASYNC.
|
| 1856 | *
|
| 1857 | * Only check it without removing it from the list.
|
| 1858 | * Since the folio can be on deferred_split_scan()
|
| 1859 | * local list and removing it can cause the local list
|
| 1860 | * corruption. Folio split process below can handle it
|
| 1861 | * with the help of folio_ref_freeze().
|
| 1862 | *
|
| 1863 | * nr_pages > 2 is needed to avoid checking order-1
|
| 1864 | * page cache folios. They exist, in contrast to
|
| 1865 | * non-existent order-1 anonymous folios, and do not
|
| 1866 | * use _deferred_list.
|
| 1867 |
|
| 1868 | if (nr_pages > 2 &&
|
| 1869 | !list_empty(&folio->_deferred_list) &&
|
| 1870 | folio_test_partially_mapped(folio)) {
|
| 1871 | if (!try_split_folio(folio, split_folios, mode)) {
|
| 1872 | nr_failed++;
|
| 1873 | stats->nr_thp_failed += is_thp;
|
| 1874 | stats->nr_thp_split += is_thp;
|
| 1875 | stats->nr_split++;
|
| 1876 | continue;
|
| 1877 | }
|
| 1878 | }
|
| 1879 |
|
| 1880 | |
| 1881 | * Large folio migration might be unsupported or
|
| 1882 | * the allocation might be failed so we should retry
|
| 1883 | * on the same folio with the large folio split
|
| 1884 | * to normal folios.
|
| 1885 | *
|
| 1886 | * Split folios are put in split_folios, and
|
| 1887 | * we will migrate them after the rest of the
|
| 1888 | * list is processed.
|
| 1889 |
|
| 1890 | if (!thp_migration_supported() && is_thp) {
|
| 1891 | nr_failed++;
|
| 1892 | stats->nr_thp_failed++;
|
| 1893 | if (!try_split_folio(folio, split_folios, mode)) {
|
| 1894 | stats->nr_thp_split++;
|
| 1895 | stats->nr_split++;
|
| 1896 | continue;
|
| 1897 | }
|
| 1898 | stats->nr_failed_pages += nr_pages;
|
| 1899 | list_move_tail(&folio->lru, ret_folios);
|
| 1900 | continue;
|
| 1901 | }
|
| 1902 |
|
| 1903 | |
| 1904 | * If we are holding the last folio reference, the folio
|
| 1905 | * was freed from under us, so just drop our reference.
|
| 1906 |
|
| 1907 | if (likely(!page_has_movable_ops(&folio->page)) &&
|
| 1908 | folio_ref_count(folio) == 1) {
|
| 1909 | folio_clear_active(folio);
|
| 1910 | folio_clear_unevictable(folio);
|
| 1911 | list_del(&folio->lru);
|
| 1912 | migrate_folio_done(folio, reason);
|
| 1913 | stats->nr_succeeded += nr_pages;
|
| 1914 | stats->nr_thp_succeeded += is_thp;
|
| 1915 | continue;
|
| 1916 | }
|
| 1917 |
|
| 1918 | rc = migrate_folio_unmap(get_new_folio, put_new_folio,
|
| 1919 | private, folio, &dst, mode, ret_folios);
|
| 1920 | |
| 1921 | * The rules are:
|
| 1922 | * 0: folio will be put on unmap_folios list,
|
| 1923 | * dst folio put on dst_folios list
|
| 1924 | * -EAGAIN: stay on the from list
|
| 1925 | * -ENOMEM: stay on the from list
|
| 1926 | * Other errno: put on ret_folios list
|
| 1927 |
|
| 1928 | switch(rc) {
|
| 1929 | case -ENOMEM:
|
| 1930 | |
| 1931 | * When memory is low, don't bother to try to migrate
|
| 1932 | * other folios, move unmapped folios, then exit.
|
| 1933 |
|
| 1934 | nr_failed++;
|
| 1935 | stats->nr_thp_failed += is_thp;
|
| 1936 |
|
| 1937 | if (is_large && !nosplit) {
|
| 1938 | int ret = try_split_folio(folio, split_folios, mode);
|
| 1939 |
|
| 1940 | if (!ret) {
|
| 1941 | stats->nr_thp_split += is_thp;
|
| 1942 | stats->nr_split++;
|
| 1943 | break;
|
| 1944 | } else if (reason == MR_LONGTERM_PIN &&
|
| 1945 | ret == -EAGAIN) {
|
| 1946 | |
| 1947 | * Try again to split large folio to
|
| 1948 | * mitigate the failure of longterm pinning.
|
| 1949 |
|
| 1950 | retry++;
|
| 1951 | thp_retry += is_thp;
|
| 1952 | nr_retry_pages += nr_pages;
|
| 1953 |
|
| 1954 | nr_failed--;
|
| 1955 | stats->nr_thp_failed -= is_thp;
|
| 1956 | break;
|
| 1957 | }
|
| 1958 | }
|
| 1959 |
|
| 1960 | stats->nr_failed_pages += nr_pages + nr_retry_pages;
|
| 1961 |
|
| 1962 | stats->nr_thp_failed += thp_retry;
|
| 1963 | rc_saved = rc;
|
| 1964 | if (list_empty(&unmap_folios))
|
| 1965 | goto out;
|
| 1966 | else
|
| 1967 | goto move;
|
| 1968 | case -EAGAIN:
|
| 1969 | retry++;
|
| 1970 | thp_retry += is_thp;
|
| 1971 | nr_retry_pages += nr_pages;
|
| 1972 | break;
|
| 1973 | case 0:
|
| 1974 | list_move_tail(&folio->lru, &unmap_folios);
|
| 1975 | list_add_tail(&dst->lru, &dst_folios);
|
| 1976 | break;
|
| 1977 | default:
|
| 1978 | |
| 1979 | * Permanent failure (-EBUSY, etc.):
|
| 1980 | * unlike -EAGAIN case, the failed folio is
|
| 1981 | * removed from migration folio list and not
|
| 1982 | * retried in the next outer loop.
|
| 1983 |
|
| 1984 | nr_failed++;
|
| 1985 | stats->nr_thp_failed += is_thp;
|
| 1986 | stats->nr_failed_pages += nr_pages;
|
| 1987 | break;
|
| 1988 | }
|
| 1989 | }
|
| 1990 | }
|
| 1991 | nr_failed += retry;
|
| 1992 | stats->nr_thp_failed += thp_retry;
|
| 1993 | stats->nr_failed_pages += nr_retry_pages;
|
| 1994 | move:
|
| 1995 |
|
| 1996 | try_to_unmap_flush();
|
| 1997 |
|
| 1998 | retry = 1;
|
| 1999 | for (pass = 0; pass < nr_pass && retry; pass++) {
|
| 2000 | retry = 0;
|
| 2001 | thp_retry = 0;
|
| 2002 | nr_retry_pages = 0;
|
| 2003 |
|
| 2004 |
|
| 2005 | migrate_folios_move(&unmap_folios, &dst_folios,
|
| 2006 | put_new_folio, private, mode, reason,
|
| 2007 | ret_folios, stats, &retry, &thp_retry,
|
| 2008 | &nr_failed, &nr_retry_pages);
|
| 2009 | }
|
| 2010 | nr_failed += retry;
|
| 2011 | stats->nr_thp_failed += thp_retry;
|
| 2012 | stats->nr_failed_pages += nr_retry_pages;
|
| 2013 |
|
| 2014 | rc = rc_saved ? : nr_failed;
|
| 2015 | out:
|
| 2016 |
|
| 2017 | migrate_folios_undo(&unmap_folios, &dst_folios,
|
| 2018 | put_new_folio, private, ret_folios);
|
| 2019 |
|
| 2020 | return rc;
|
| 2021 | }
|
| 2022 |
|
| 2023 | static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio,
|
| 2024 | free_folio_t put_new_folio, unsigned long private,
|
| 2025 | enum migrate_mode mode, enum migrate_reason reason,
|
| 2026 | struct list_head *ret_folios, struct list_head *split_folios,
|
| 2027 | struct migrate_pages_stats *stats)
|
| 2028 | {
|
| 2029 | int rc, nr_failed = 0;
|
| 2030 | LIST_HEAD(folios);
|
| 2031 | struct migrate_pages_stats astats;
|
| 2032 |
|
| 2033 | memset(&astats, 0, sizeof(astats));
|
| 2034 |
|
| 2035 | rc = migrate_pages_batch(from, get_new_folio, put_new_folio, private, MIGRATE_ASYNC,
|
| 2036 | reason, &folios, split_folios, &astats,
|
| 2037 | NR_MAX_MIGRATE_ASYNC_RETRY);
|
| 2038 | stats->nr_succeeded += astats.nr_succeeded;
|
| 2039 | stats->nr_thp_succeeded += astats.nr_thp_succeeded;
|
| 2040 | stats->nr_thp_split += astats.nr_thp_split;
|
| 2041 | stats->nr_split += astats.nr_split;
|
| 2042 | if (rc < 0) {
|
| 2043 | stats->nr_failed_pages += astats.nr_failed_pages;
|
| 2044 | stats->nr_thp_failed += astats.nr_thp_failed;
|
| 2045 | list_splice_tail(&folios, ret_folios);
|
| 2046 | return rc;
|
| 2047 | }
|
| 2048 | stats->nr_thp_failed += astats.nr_thp_split;
|
| 2049 | |
| 2050 | * Do not count rc, as pages will be retried below.
|
| 2051 | * Count nr_split only, since it includes nr_thp_split.
|
| 2052 |
|
| 2053 | nr_failed += astats.nr_split;
|
| 2054 | |
| 2055 | * Fall back to migrate all failed folios one by one synchronously. All
|
| 2056 | * failed folios except split THPs will be retried, so their failure
|
| 2057 | * isn't counted
|
| 2058 |
|
| 2059 | list_splice_tail_init(&folios, from);
|
| 2060 | while (!list_empty(from)) {
|
| 2061 | list_move(from->next, &folios);
|
| 2062 | rc = migrate_pages_batch(&folios, get_new_folio, put_new_folio,
|
| 2063 | private, mode, reason, ret_folios,
|
| 2064 | split_folios, stats, NR_MAX_MIGRATE_SYNC_RETRY);
|
| 2065 | list_splice_tail_init(&folios, ret_folios);
|
| 2066 | if (rc < 0)
|
| 2067 | return rc;
|
| 2068 | nr_failed += rc;
|
| 2069 | }
|
| 2070 |
|
| 2071 | return nr_failed;
|
| 2072 | }
|
| 2073 |
|
| 2074 | |
| 2075 | * migrate_pages - migrate the folios specified in a list, to the free folios
|
| 2076 | * supplied as the target for the page migration
|
| 2077 | *
|
| 2078 | * @from: The list of folios to be migrated.
|
| 2079 | * @get_new_folio: The function used to allocate free folios to be used
|
| 2080 | * as the target of the folio migration.
|
| 2081 | * @put_new_folio: The function used to free target folios if migration
|
| 2082 | * fails, or NULL if no special handling is necessary.
|
| 2083 | * @private: Private data to be passed on to get_new_folio()
|
| 2084 | * @mode: The migration mode that specifies the constraints for
|
| 2085 | * folio migration, if any.
|
| 2086 | * @reason: The reason for folio migration.
|
| 2087 | * @ret_succeeded: Set to the number of folios migrated successfully if
|
| 2088 | * the caller passes a non-NULL pointer.
|
| 2089 | *
|
| 2090 | * The function returns after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no folios
|
| 2091 | * are movable any more because the list has become empty or no retryable folios
|
| 2092 | * exist any more. It is caller's responsibility to call putback_movable_pages()
|
| 2093 | * only if ret != 0.
|
| 2094 | *
|
| 2095 | * Returns the number of {normal folio, large folio, hugetlb} that were not
|
| 2096 | * migrated, or an error code. The number of large folio splits will be
|
| 2097 | * considered as the number of non-migrated large folio, no matter how many
|
| 2098 | * split folios of the large folio are migrated successfully.
|
| 2099 |
|
| 2100 | int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
|
| 2101 | free_folio_t put_new_folio, unsigned long private,
|
| 2102 | enum migrate_mode mode, enum migrate_reason reason, unsigned int *ret_succeeded)
|
| 2103 | {
|
| 2104 | int rc, rc_gather;
|
| 2105 | int nr_pages;
|
| 2106 | struct folio *folio, *folio2;
|
| 2107 | LIST_HEAD(folios);
|
| 2108 | LIST_HEAD(ret_folios);
|
| 2109 | LIST_HEAD(split_folios);
|
| 2110 | struct migrate_pages_stats stats;
|
| 2111 |
|
| 2112 | trace_mm_migrate_pages_start(mode, reason);
|
| 2113 |
|
| 2114 | memset(&stats, 0, sizeof(stats));
|
| 2115 |
|
| 2116 | rc_gather = migrate_hugetlbs(from, get_new_folio, put_new_folio, private,
|
| 2117 | mode, reason, &stats, &ret_folios);
|
| 2118 | if (rc_gather < 0)
|
| 2119 | goto out;
|
| 2120 |
|
| 2121 | again:
|
| 2122 | nr_pages = 0;
|
| 2123 | list_for_each_entry_safe(folio, folio2, from, lru) {
|
| 2124 |
|
| 2125 | if (folio_test_hugetlb(folio)) {
|
| 2126 | list_move_tail(&folio->lru, &ret_folios);
|
| 2127 | continue;
|
| 2128 | }
|
| 2129 |
|
| 2130 | nr_pages += folio_nr_pages(folio);
|
| 2131 | if (nr_pages >= NR_MAX_BATCHED_MIGRATION)
|
| 2132 | break;
|
| 2133 | }
|
| 2134 | if (nr_pages >= NR_MAX_BATCHED_MIGRATION)
|
| 2135 | list_cut_before(&folios, from, &folio2->lru);
|
| 2136 | else
|
| 2137 | list_splice_init(from, &folios);
|
| 2138 | if (mode == MIGRATE_ASYNC)
|
| 2139 | rc = migrate_pages_batch(&folios, get_new_folio, put_new_folio,
|
| 2140 | private, mode, reason, &ret_folios,
|
| 2141 | &split_folios, &stats,
|
| 2142 | NR_MAX_MIGRATE_PAGES_RETRY);
|
| 2143 | else
|
| 2144 | rc = migrate_pages_sync(&folios, get_new_folio, put_new_folio,
|
| 2145 | private, mode, reason, &ret_folios,
|
| 2146 | &split_folios, &stats);
|
| 2147 | list_splice_tail_init(&folios, &ret_folios);
|
| 2148 | if (rc < 0) {
|
| 2149 | rc_gather = rc;
|
| 2150 | list_splice_tail(&split_folios, &ret_folios);
|
| 2151 | goto out;
|
| 2152 | }
|
| 2153 | if (!list_empty(&split_folios)) {
|
| 2154 | |
| 2155 | * Failure isn't counted since all split folios of a large folio
|
| 2156 | * is counted as 1 failure already. And, we only try to migrate
|
| 2157 | * with minimal effort, force MIGRATE_ASYNC mode and retry once.
|
| 2158 |
|
| 2159 | migrate_pages_batch(&split_folios, get_new_folio,
|
| 2160 | put_new_folio, private, MIGRATE_ASYNC, reason,
|
| 2161 | &ret_folios, NULL, &stats, 1);
|
| 2162 | list_splice_tail_init(&split_folios, &ret_folios);
|
| 2163 | }
|
| 2164 | rc_gather += rc;
|
| 2165 | if (!list_empty(from))
|
| 2166 | goto again;
|
| 2167 | out:
|
| 2168 | |
| 2169 | * Put the permanent failure folio back to migration list, they
|
| 2170 | * will be put back to the right list by the caller.
|
| 2171 |
|
| 2172 | list_splice(&ret_folios, from);
|
| 2173 |
|
| 2174 | |
| 2175 | * Return 0 in case all split folios of fail-to-migrate large folios
|
| 2176 | * are migrated successfully.
|
| 2177 |
|
| 2178 | if (list_empty(from))
|
| 2179 | rc_gather = 0;
|
| 2180 |
|
| 2181 | count_vm_events(PGMIGRATE_SUCCESS, stats.nr_succeeded);
|
| 2182 | count_vm_events(PGMIGRATE_FAIL, stats.nr_failed_pages);
|
| 2183 | count_vm_events(THP_MIGRATION_SUCCESS, stats.nr_thp_succeeded);
|
| 2184 | count_vm_events(THP_MIGRATION_FAIL, stats.nr_thp_failed);
|
| 2185 | count_vm_events(THP_MIGRATION_SPLIT, stats.nr_thp_split);
|
| 2186 | trace_mm_migrate_pages(stats.nr_succeeded, stats.nr_failed_pages,
|
| 2187 | stats.nr_thp_succeeded, stats.nr_thp_failed,
|
| 2188 | stats.nr_thp_split, stats.nr_split, mode,
|
| 2189 | reason);
|
| 2190 |
|
| 2191 | if (ret_succeeded)
|
| 2192 | *ret_succeeded = stats.nr_succeeded;
|
| 2193 |
|
| 2194 | return rc_gather;
|
| 2195 | }
|
| 2196 |
|
| 2197 | struct folio *alloc_migration_target(struct folio *src, unsigned long private)
|
| 2198 | {
|
| 2199 | struct migration_target_control *mtc;
|
| 2200 | gfp_t gfp_mask;
|
| 2201 | unsigned int order = 0;
|
| 2202 | int nid;
|
| 2203 | enum zone_type zidx;
|
| 2204 |
|
| 2205 | mtc = (struct migration_target_control *)private;
|
| 2206 | gfp_mask = mtc->gfp_mask;
|
| 2207 | nid = mtc->nid;
|
| 2208 | if (nid == NUMA_NO_NODE)
|
| 2209 | nid = folio_nid(src);
|
| 2210 |
|
| 2211 | if (folio_test_hugetlb(src)) {
|
| 2212 | struct hstate *h = folio_hstate(src);
|
| 2213 |
|
| 2214 | gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
|
| 2215 | return alloc_hugetlb_folio_nodemask(h, nid,
|
| 2216 | mtc->nmask, gfp_mask,
|
| 2217 | htlb_allow_alloc_fallback(mtc->reason));
|
| 2218 | }
|
| 2219 |
|
| 2220 | if (folio_test_large(src)) {
|
| 2221 | |
| 2222 | * clear __GFP_RECLAIM to make the migration callback
|
| 2223 | * consistent with regular THP allocations.
|
| 2224 |
|
| 2225 | gfp_mask &= ~__GFP_RECLAIM;
|
| 2226 | gfp_mask |= GFP_TRANSHUGE;
|
| 2227 | order = folio_order(src);
|
| 2228 | }
|
| 2229 | zidx = folio_zonenum(src);
|
| 2230 | if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
|
| 2231 | gfp_mask |= __GFP_HIGHMEM;
|
| 2232 |
|
| 2233 | return __folio_alloc(gfp_mask, order, nid, mtc->nmask);
|
| 2234 | }
|
| 2235 |
|
| 2236 | #ifdef CONFIG_NUMA_MIGRATION
|
| 2237 | static int store_status(int __user *status, int start, int value, int nr)
|
| 2238 | {
|
| 2239 | while (nr-- > 0) {
|
| 2240 | if (put_user(value, status + start))
|
| 2241 | return -EFAULT;
|
| 2242 | start++;
|
| 2243 | }
|
| 2244 |
|
| 2245 | return 0;
|
| 2246 | }
|
| 2247 |
|
| 2248 | static int do_move_pages_to_node(struct list_head *pagelist, int node)
|
| 2249 | {
|
| 2250 | int err;
|
| 2251 | struct migration_target_control mtc = {
|
| 2252 | .nid = node,
|
| 2253 | .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
|
| 2254 | .reason = MR_SYSCALL,
|
| 2255 | };
|
| 2256 |
|
| 2257 | err = migrate_pages(pagelist, alloc_migration_target, NULL,
|
| 2258 | (unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
|
| 2259 | if (err)
|
| 2260 | putback_movable_pages(pagelist);
|
| 2261 | return err;
|
| 2262 | }
|
| 2263 |
|
| 2264 | static int __add_folio_for_migration(struct folio *folio, int node,
|
| 2265 | struct list_head *pagelist, bool migrate_all)
|
| 2266 | {
|
| 2267 | if (is_zero_folio(folio) || is_huge_zero_folio(folio))
|
| 2268 | return -EFAULT;
|
| 2269 |
|
| 2270 | if (folio_is_zone_device(folio))
|
| 2271 | return -ENOENT;
|
| 2272 |
|
| 2273 | if (folio_nid(folio) == node)
|
| 2274 | return 0;
|
| 2275 |
|
| 2276 | if (folio_maybe_mapped_shared(folio) && !migrate_all)
|
| 2277 | return -EACCES;
|
| 2278 |
|
| 2279 | if (folio_test_hugetlb(folio)) {
|
| 2280 | if (folio_isolate_hugetlb(folio, pagelist))
|
| 2281 | return 1;
|
| 2282 | } else if (folio_isolate_lru(folio)) {
|
| 2283 | list_add_tail(&folio->lru, pagelist);
|
| 2284 | node_stat_mod_folio(folio,
|
| 2285 | NR_ISOLATED_ANON + folio_is_file_lru(folio),
|
| 2286 | folio_nr_pages(folio));
|
| 2287 | return 1;
|
| 2288 | }
|
| 2289 | return -EBUSY;
|
| 2290 | }
|
| 2291 |
|
| 2292 | |
| 2293 | * Resolves the given address to a struct folio, isolates it from the LRU and
|
| 2294 | * puts it to the given pagelist.
|
| 2295 | * Returns:
|
| 2296 | * errno - if the folio cannot be found/isolated
|
| 2297 | * 0 - when it doesn't have to be migrated because it is already on the
|
| 2298 | * target node
|
| 2299 | * 1 - when it has been queued
|
| 2300 |
|
| 2301 | static int add_folio_for_migration(struct mm_struct *mm, const void __user *p,
|
| 2302 | int node, struct list_head *pagelist, bool migrate_all)
|
| 2303 | {
|
| 2304 | struct vm_area_struct *vma;
|
| 2305 | struct folio_walk fw;
|
| 2306 | struct folio *folio;
|
| 2307 | unsigned long addr;
|
| 2308 | int err = -EFAULT;
|
| 2309 |
|
| 2310 | mmap_read_lock(mm);
|
| 2311 | addr = (unsigned long)untagged_addr_remote(mm, p);
|
| 2312 |
|
| 2313 | vma = vma_lookup(mm, addr);
|
| 2314 | if (vma && vma_migratable(vma)) {
|
| 2315 | folio = folio_walk_start(&fw, vma, addr, FW_ZEROPAGE);
|
| 2316 | if (folio) {
|
| 2317 | err = __add_folio_for_migration(folio, node, pagelist,
|
| 2318 | migrate_all);
|
| 2319 | folio_walk_end(&fw, vma);
|
| 2320 | } else {
|
| 2321 | err = -ENOENT;
|
| 2322 | }
|
| 2323 | }
|
| 2324 | mmap_read_unlock(mm);
|
| 2325 | return err;
|
| 2326 | }
|
| 2327 |
|
| 2328 | static int move_pages_and_store_status(int node,
|
| 2329 | struct list_head *pagelist, int __user *status,
|
| 2330 | int start, int i, unsigned long nr_pages)
|
| 2331 | {
|
| 2332 | int err;
|
| 2333 |
|
| 2334 | if (list_empty(pagelist))
|
| 2335 | return 0;
|
| 2336 |
|
| 2337 | err = do_move_pages_to_node(pagelist, node);
|
| 2338 | if (err) {
|
| 2339 | |
| 2340 | * Positive err means the number of failed
|
| 2341 | * pages to migrate. Since we are going to
|
| 2342 | * abort and return the number of non-migrated
|
| 2343 | * pages, so need to include the rest of the
|
| 2344 | * nr_pages that have not been attempted as
|
| 2345 | * well.
|
| 2346 |
|
| 2347 | if (err > 0)
|
| 2348 | err += nr_pages - i;
|
| 2349 | return err;
|
| 2350 | }
|
| 2351 | return store_status(status, start, node, i - start);
|
| 2352 | }
|
| 2353 |
|
| 2354 | |
| 2355 | * Migrate an array of page address onto an array of nodes and fill
|
| 2356 | * the corresponding array of status.
|
| 2357 |
|
| 2358 | static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
|
| 2359 | unsigned long nr_pages,
|
| 2360 | const void __user * __user *pages,
|
| 2361 | const int __user *nodes,
|
| 2362 | int __user *status, int flags)
|
| 2363 | {
|
| 2364 | compat_uptr_t __user *compat_pages = (void __user *)pages;
|
| 2365 | int current_node = NUMA_NO_NODE;
|
| 2366 | LIST_HEAD(pagelist);
|
| 2367 | int start, i;
|
| 2368 | int err = 0, err1;
|
| 2369 |
|
| 2370 | lru_cache_disable();
|
| 2371 |
|
| 2372 | for (i = start = 0; i < nr_pages; i++) {
|
| 2373 | const void __user *p;
|
| 2374 | int node;
|
| 2375 |
|
| 2376 | err = -EFAULT;
|
| 2377 | if (in_compat_syscall()) {
|
| 2378 | compat_uptr_t cp;
|
| 2379 |
|
| 2380 | if (get_user(cp, compat_pages + i))
|
| 2381 | goto out_flush;
|
| 2382 |
|
| 2383 | p = compat_ptr(cp);
|
| 2384 | } else {
|
| 2385 | if (get_user(p, pages + i))
|
| 2386 | goto out_flush;
|
| 2387 | }
|
| 2388 | if (get_user(node, nodes + i))
|
| 2389 | goto out_flush;
|
| 2390 |
|
| 2391 | err = -ENODEV;
|
| 2392 | if (node < 0 || node >= MAX_NUMNODES)
|
| 2393 | goto out_flush;
|
| 2394 | if (!node_state(node, N_MEMORY))
|
| 2395 | goto out_flush;
|
| 2396 |
|
| 2397 | err = -EACCES;
|
| 2398 | if (!node_isset(node, task_nodes))
|
| 2399 | goto out_flush;
|
| 2400 |
|
| 2401 | if (current_node == NUMA_NO_NODE) {
|
| 2402 | current_node = node;
|
| 2403 | start = i;
|
| 2404 | } else if (node != current_node) {
|
| 2405 | err = move_pages_and_store_status(current_node,
|
| 2406 | &pagelist, status, start, i, nr_pages);
|
| 2407 | if (err)
|
| 2408 | goto out;
|
| 2409 | start = i;
|
| 2410 | current_node = node;
|
| 2411 | }
|
| 2412 |
|
| 2413 | |
| 2414 | * Errors in the page lookup or isolation are not fatal and we simply
|
| 2415 | * report them via status
|
| 2416 |
|
| 2417 | err = add_folio_for_migration(mm, p, current_node, &pagelist,
|
| 2418 | flags & MPOL_MF_MOVE_ALL);
|
| 2419 |
|
| 2420 | if (err > 0) {
|
| 2421 |
|
| 2422 | continue;
|
| 2423 | }
|
| 2424 |
|
| 2425 | |
| 2426 | * If the page is already on the target node (!err), store the
|
| 2427 | * node, otherwise, store the err.
|
| 2428 |
|
| 2429 | err = store_status(status, i, err ? : current_node, 1);
|
| 2430 | if (err)
|
| 2431 | goto out_flush;
|
| 2432 |
|
| 2433 | err = move_pages_and_store_status(current_node, &pagelist,
|
| 2434 | status, start, i, nr_pages);
|
| 2435 | if (err) {
|
| 2436 |
|
| 2437 | if (err > 0)
|
| 2438 | err--;
|
| 2439 | goto out;
|
| 2440 | }
|
| 2441 | current_node = NUMA_NO_NODE;
|
| 2442 | }
|
| 2443 | out_flush:
|
| 2444 |
|
| 2445 | err1 = move_pages_and_store_status(current_node, &pagelist,
|
| 2446 | status, start, i, nr_pages);
|
| 2447 | if (err >= 0)
|
| 2448 | err = err1;
|
| 2449 | out:
|
| 2450 | lru_cache_enable();
|
| 2451 | return err;
|
| 2452 | }
|
| 2453 |
|
| 2454 | |
| 2455 | * Determine the nodes of an array of pages and store it in an array of status.
|
| 2456 |
|
| 2457 | static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
|
| 2458 | const void __user **pages, int *status)
|
| 2459 | {
|
| 2460 | unsigned long i;
|
| 2461 |
|
| 2462 | mmap_read_lock(mm);
|
| 2463 |
|
| 2464 | for (i = 0; i < nr_pages; i++) {
|
| 2465 | unsigned long addr = (unsigned long)(*pages);
|
| 2466 | struct vm_area_struct *vma;
|
| 2467 | struct folio_walk fw;
|
| 2468 | struct folio *folio;
|
| 2469 | int err = -EFAULT;
|
| 2470 |
|
| 2471 | vma = vma_lookup(mm, addr);
|
| 2472 | if (!vma)
|
| 2473 | goto set_status;
|
| 2474 |
|
| 2475 | folio = folio_walk_start(&fw, vma, addr, FW_ZEROPAGE);
|
| 2476 | if (folio) {
|
| 2477 | if (is_zero_folio(folio) || is_huge_zero_folio(folio))
|
| 2478 | err = -EFAULT;
|
| 2479 | else if (folio_is_zone_device(folio))
|
| 2480 | err = -ENOENT;
|
| 2481 | else
|
| 2482 | err = folio_nid(folio);
|
| 2483 | folio_walk_end(&fw, vma);
|
| 2484 | } else {
|
| 2485 | err = -ENOENT;
|
| 2486 | }
|
| 2487 | set_status:
|
| 2488 | *status = err;
|
| 2489 |
|
| 2490 | pages++;
|
| 2491 | status++;
|
| 2492 | }
|
| 2493 |
|
| 2494 | mmap_read_unlock(mm);
|
| 2495 | }
|
| 2496 |
|
| 2497 | static int get_compat_pages_array(const void __user *chunk_pages[],
|
| 2498 | const void __user * __user *pages,
|
| 2499 | unsigned long chunk_offset,
|
| 2500 | unsigned long chunk_nr)
|
| 2501 | {
|
| 2502 | compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages;
|
| 2503 | compat_uptr_t p;
|
| 2504 | int i;
|
| 2505 |
|
| 2506 | for (i = 0; i < chunk_nr; i++) {
|
| 2507 | if (get_user(p, pages32 + chunk_offset + i))
|
| 2508 | return -EFAULT;
|
| 2509 | chunk_pages[i] = compat_ptr(p);
|
| 2510 | }
|
| 2511 |
|
| 2512 | return 0;
|
| 2513 | }
|
| 2514 |
|
| 2515 | |
| 2516 | * Determine the nodes of a user array of pages and store it in
|
| 2517 | * a user array of status.
|
| 2518 |
|
| 2519 | static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
|
| 2520 | const void __user * __user *pages,
|
| 2521 | int __user *status)
|
| 2522 | {
|
| 2523 | #define DO_PAGES_STAT_CHUNK_NR 16UL
|
| 2524 | const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
|
| 2525 | int chunk_status[DO_PAGES_STAT_CHUNK_NR];
|
| 2526 | unsigned long chunk_offset = 0;
|
| 2527 |
|
| 2528 | while (nr_pages) {
|
| 2529 | unsigned long chunk_nr = min(nr_pages, DO_PAGES_STAT_CHUNK_NR);
|
| 2530 |
|
| 2531 | if (in_compat_syscall()) {
|
| 2532 | if (get_compat_pages_array(chunk_pages, pages,
|
| 2533 | chunk_offset, chunk_nr))
|
| 2534 | break;
|
| 2535 | } else {
|
| 2536 | if (copy_from_user(chunk_pages, pages + chunk_offset,
|
| 2537 | chunk_nr * sizeof(*chunk_pages)))
|
| 2538 | break;
|
| 2539 | }
|
| 2540 |
|
| 2541 | do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
|
| 2542 |
|
| 2543 | if (copy_to_user(status + chunk_offset, chunk_status,
|
| 2544 | chunk_nr * sizeof(*status)))
|
| 2545 | break;
|
| 2546 |
|
| 2547 | chunk_offset += chunk_nr;
|
| 2548 | nr_pages -= chunk_nr;
|
| 2549 | }
|
| 2550 | return nr_pages ? -EFAULT : 0;
|
| 2551 | }
|
| 2552 |
|
| 2553 | static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
|
| 2554 | {
|
| 2555 | struct task_struct *task;
|
| 2556 | struct mm_struct *mm;
|
| 2557 |
|
| 2558 | |
| 2559 | * There is no need to check if current process has the right to modify
|
| 2560 | * the specified process when they are same.
|
| 2561 |
|
| 2562 | if (!pid) {
|
| 2563 | mmget(current->mm);
|
| 2564 | *mem_nodes = cpuset_mems_allowed(current);
|
| 2565 | return current->mm;
|
| 2566 | }
|
| 2567 |
|
| 2568 | task = find_get_task_by_vpid(pid);
|
| 2569 | if (!task)
|
| 2570 | return ERR_PTR(-ESRCH);
|
| 2571 |
|
| 2572 | if (down_read_killable(&task->signal->exec_update_lock)) {
|
| 2573 | mm = ERR_PTR(-EINTR);
|
| 2574 | goto out;
|
| 2575 | }
|
| 2576 | |
| 2577 | * Check if this process has the right to modify the specified
|
| 2578 | * process. Use the regular "ptrace_may_access()" checks.
|
| 2579 |
|
| 2580 | if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
|
| 2581 | mm = ERR_PTR(-EPERM);
|
| 2582 | goto unlock;
|
| 2583 | }
|
| 2584 |
|
| 2585 | mm = ERR_PTR(security_task_movememory(task));
|
| 2586 | if (IS_ERR(mm))
|
| 2587 | goto unlock;
|
| 2588 | *mem_nodes = cpuset_mems_allowed(task);
|
| 2589 | mm = get_task_mm(task);
|
| 2590 | unlock:
|
| 2591 | up_read(&task->signal->exec_update_lock);
|
| 2592 | out:
|
| 2593 | put_task_struct(task);
|
| 2594 | if (!mm)
|
| 2595 | mm = ERR_PTR(-EINVAL);
|
| 2596 | return mm;
|
| 2597 | }
|
| 2598 |
|
| 2599 | |
| 2600 | * Move a list of pages in the address space of the currently executing
|
| 2601 | * process.
|
| 2602 |
|
| 2603 | static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
|
| 2604 | const void __user * __user *pages,
|
| 2605 | const int __user *nodes,
|
| 2606 | int __user *status, int flags)
|
| 2607 | {
|
| 2608 | struct mm_struct *mm;
|
| 2609 | int err;
|
| 2610 | nodemask_t task_nodes;
|
| 2611 |
|
| 2612 |
|
| 2613 | if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
|
| 2614 | return -EINVAL;
|
| 2615 |
|
| 2616 | if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
|
| 2617 | return -EPERM;
|
| 2618 |
|
| 2619 | mm = find_mm_struct(pid, &task_nodes);
|
| 2620 | if (IS_ERR(mm))
|
| 2621 | return PTR_ERR(mm);
|
| 2622 |
|
| 2623 | if (nodes)
|
| 2624 | err = do_pages_move(mm, task_nodes, nr_pages, pages,
|
| 2625 | nodes, status, flags);
|
| 2626 | else
|
| 2627 | err = do_pages_stat(mm, nr_pages, pages, status);
|
| 2628 |
|
| 2629 | mmput(mm);
|
| 2630 | return err;
|
| 2631 | }
|
| 2632 |
|
| 2633 | SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
|
| 2634 | const void __user * __user *, pages,
|
| 2635 | const int __user *, nodes,
|
| 2636 | int __user *, status, int, flags)
|
| 2637 | {
|
| 2638 | return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
|
| 2639 | }
|
| 2640 | #endif
|
| 2641 |
|
| 2642 | #ifdef CONFIG_NUMA_BALANCING
|
| 2643 | |
| 2644 | * Returns true if this is a safe migration target node for misplaced NUMA
|
| 2645 | * pages. Currently it only checks the watermarks which is crude.
|
| 2646 |
|
| 2647 | static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
|
| 2648 | unsigned long nr_migrate_pages)
|
| 2649 | {
|
| 2650 | int z;
|
| 2651 |
|
| 2652 | for (z = pgdat->nr_zones - 1; z >= 0; z--) {
|
| 2653 | struct zone *zone = pgdat->node_zones + z;
|
| 2654 |
|
| 2655 | if (!managed_zone(zone))
|
| 2656 | continue;
|
| 2657 |
|
| 2658 |
|
| 2659 | if (!zone_watermark_ok(zone, 0,
|
| 2660 | high_wmark_pages(zone) +
|
| 2661 | nr_migrate_pages,
|
| 2662 | ZONE_MOVABLE, ALLOC_CMA))
|
| 2663 | continue;
|
| 2664 | return true;
|
| 2665 | }
|
| 2666 | return false;
|
| 2667 | }
|
| 2668 |
|
| 2669 | static struct folio *alloc_misplaced_dst_folio(struct folio *src,
|
| 2670 | unsigned long data)
|
| 2671 | {
|
| 2672 | int nid = (int) data;
|
| 2673 | int order = folio_order(src);
|
| 2674 | gfp_t gfp = __GFP_THISNODE;
|
| 2675 |
|
| 2676 | if (order > 0)
|
| 2677 | gfp |= GFP_TRANSHUGE_LIGHT;
|
| 2678 | else {
|
| 2679 | gfp |= GFP_HIGHUSER_MOVABLE | __GFP_NOMEMALLOC | __GFP_NORETRY |
|
| 2680 | __GFP_NOWARN;
|
| 2681 | gfp &= ~__GFP_RECLAIM;
|
| 2682 | }
|
| 2683 | return __folio_alloc_node(gfp, order, nid);
|
| 2684 | }
|
| 2685 |
|
| 2686 | |
| 2687 | * Prepare for calling migrate_misplaced_folio() by isolating the folio if
|
| 2688 | * permitted. Must be called with the PTL still held.
|
| 2689 |
|
| 2690 | int migrate_misplaced_folio_prepare(struct folio *folio,
|
| 2691 | struct vm_area_struct *vma, int node)
|
| 2692 | {
|
| 2693 | int nr_pages = folio_nr_pages(folio);
|
| 2694 | pg_data_t *pgdat = NODE_DATA(node);
|
| 2695 |
|
| 2696 | if (folio_is_file_lru(folio)) {
|
| 2697 | |
| 2698 | * Do not migrate file folios that are mapped in multiple
|
| 2699 | * processes with execute permissions as they are probably
|
| 2700 | * shared libraries.
|
| 2701 | *
|
| 2702 | * See folio_maybe_mapped_shared() on possible imprecision
|
| 2703 | * when we cannot easily detect if a folio is shared.
|
| 2704 |
|
| 2705 | if ((vma->vm_flags & VM_EXEC) && folio_maybe_mapped_shared(folio))
|
| 2706 | return -EACCES;
|
| 2707 |
|
| 2708 | |
| 2709 | * Do not migrate dirty folios as not all filesystems can move
|
| 2710 | * dirty folios in MIGRATE_ASYNC mode which is a waste of
|
| 2711 | * cycles.
|
| 2712 |
|
| 2713 | if (folio_test_dirty(folio))
|
| 2714 | return -EAGAIN;
|
| 2715 | }
|
| 2716 |
|
| 2717 |
|
| 2718 | if (!migrate_balanced_pgdat(pgdat, nr_pages)) {
|
| 2719 | int z;
|
| 2720 |
|
| 2721 | if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING))
|
| 2722 | return -EAGAIN;
|
| 2723 | for (z = pgdat->nr_zones - 1; z >= 0; z--) {
|
| 2724 | if (managed_zone(pgdat->node_zones + z))
|
| 2725 | break;
|
| 2726 | }
|
| 2727 |
|
| 2728 | |
| 2729 | * If there are no managed zones, it should not proceed
|
| 2730 | * further.
|
| 2731 |
|
| 2732 | if (z < 0)
|
| 2733 | return -EAGAIN;
|
| 2734 |
|
| 2735 | wakeup_kswapd(pgdat->node_zones + z, 0,
|
| 2736 | folio_order(folio), ZONE_MOVABLE);
|
| 2737 | return -EAGAIN;
|
| 2738 | }
|
| 2739 |
|
| 2740 | if (!folio_isolate_lru(folio))
|
| 2741 | return -EAGAIN;
|
| 2742 |
|
| 2743 | node_stat_mod_folio(folio, NR_ISOLATED_ANON + folio_is_file_lru(folio),
|
| 2744 | nr_pages);
|
| 2745 | return 0;
|
| 2746 | }
|
| 2747 |
|
| 2748 | |
| 2749 | * Attempt to migrate a misplaced folio to the specified destination
|
| 2750 | * node. Caller is expected to have isolated the folio by calling
|
| 2751 | * migrate_misplaced_folio_prepare(), which will result in an
|
| 2752 | * elevated reference count on the folio. This function will un-isolate the
|
| 2753 | * folio, dereferencing the folio before returning.
|
| 2754 |
|
| 2755 | int migrate_misplaced_folio(struct folio *folio, int node)
|
| 2756 | {
|
| 2757 | pg_data_t *pgdat = NODE_DATA(node);
|
| 2758 | int nr_remaining;
|
| 2759 | unsigned int nr_succeeded;
|
| 2760 | LIST_HEAD(migratepages);
|
| 2761 | struct mem_cgroup *memcg = get_mem_cgroup_from_folio(folio);
|
| 2762 | struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
|
| 2763 |
|
| 2764 | list_add(&folio->lru, &migratepages);
|
| 2765 | nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_folio,
|
| 2766 | NULL, node, MIGRATE_ASYNC,
|
| 2767 | MR_NUMA_MISPLACED, &nr_succeeded);
|
| 2768 | if (nr_remaining && !list_empty(&migratepages))
|
| 2769 | putback_movable_pages(&migratepages);
|
| 2770 | if (nr_succeeded) {
|
| 2771 | count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_succeeded);
|
| 2772 | count_memcg_events(memcg, NUMA_PAGE_MIGRATE, nr_succeeded);
|
| 2773 | if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
|
| 2774 | && !node_is_toptier(folio_nid(folio))
|
| 2775 | && node_is_toptier(node))
|
| 2776 | mod_lruvec_state(lruvec, PGPROMOTE_SUCCESS, nr_succeeded);
|
| 2777 | }
|
| 2778 | mem_cgroup_put(memcg);
|
| 2779 | BUG_ON(!list_empty(&migratepages));
|
| 2780 | return nr_remaining ? -EAGAIN : 0;
|
| 2781 | }
|
| 2782 | #endif
|