| 1 | |
| 2 | |
| 3 | * linux/mm/page_isolation.c
|
| 4 |
|
| 5 |
|
| 6 | #include <linux/mm.h>
|
| 7 | #include <linux/page-isolation.h>
|
| 8 | #include <linux/pageblock-flags.h>
|
| 9 | #include <linux/memory.h>
|
| 10 | #include <linux/hugetlb.h>
|
| 11 | #include <linux/page_owner.h>
|
| 12 | #include <linux/migrate.h>
|
| 13 | #include "internal.h"
|
| 14 | #include "page_alloc.h"
|
| 15 |
|
| 16 | #define CREATE_TRACE_POINTS
|
| 17 | #include <trace/events/page_isolation.h>
|
| 18 |
|
| 19 | bool page_is_unmovable(struct zone *zone, struct page *page,
|
| 20 | enum pb_isolate_mode mode, unsigned long *step)
|
| 21 | {
|
| 22 | |
| 23 | * Both, bootmem allocations and memory holes are marked
|
| 24 | * PG_reserved and are unmovable. We can even have unmovable
|
| 25 | * allocations inside ZONE_MOVABLE, for example when
|
| 26 | * specifying "movablecore".
|
| 27 |
|
| 28 | if (PageReserved(page))
|
| 29 | return true;
|
| 30 |
|
| 31 | |
| 32 | * If the zone is movable and we have ruled out all reserved
|
| 33 | * pages then it should be reasonably safe to assume the rest
|
| 34 | * is movable.
|
| 35 |
|
| 36 | if (zone_idx(zone) == ZONE_MOVABLE)
|
| 37 | return false;
|
| 38 |
|
| 39 | |
| 40 | * Hugepages are not in LRU lists, but they're movable.
|
| 41 | * THPs are on the LRU, but need to be counted as #small pages.
|
| 42 | * We need not scan over tail pages because we don't
|
| 43 | * handle each tail page individually in migration.
|
| 44 |
|
| 45 | if (PageCompound(page)) {
|
| 46 | struct folio *folio = page_folio(page);
|
| 47 | unsigned long nr_pages, pfn;
|
| 48 | unsigned int order;
|
| 49 |
|
| 50 | order = compound_order(&folio->page);
|
| 51 | if (order > MAX_FOLIO_ORDER)
|
| 52 | return true;
|
| 53 |
|
| 54 | if (folio_test_hugetlb(folio)) {
|
| 55 | struct hstate *h;
|
| 56 |
|
| 57 | if (!IS_ENABLED(CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION))
|
| 58 | return true;
|
| 59 |
|
| 60 | |
| 61 | * The huge page may be freed so can not
|
| 62 | * use folio_hstate() directly.
|
| 63 |
|
| 64 | h = size_to_hstate(PAGE_SIZE << order);
|
| 65 | if (!h || !hugepage_migration_supported(h))
|
| 66 | return true;
|
| 67 | } else if (!PageLRU(page)) {
|
| 68 | return true;
|
| 69 | }
|
| 70 |
|
| 71 | nr_pages = 1UL << order;
|
| 72 | pfn = page_to_pfn(page);
|
| 73 | *step = (pfn | (nr_pages - 1)) + 1 - pfn;
|
| 74 | return false;
|
| 75 | }
|
| 76 |
|
| 77 | |
| 78 | * We can't use page_count without pin a page
|
| 79 | * because another CPU can free compound page.
|
| 80 | * This check already skips compound tails of THP
|
| 81 | * because their page->_refcount is zero at all time.
|
| 82 |
|
| 83 | if (!page_ref_count(page)) {
|
| 84 | if (PageBuddy(page))
|
| 85 | *step = (1 << buddy_order(page));
|
| 86 | return false;
|
| 87 | }
|
| 88 |
|
| 89 | |
| 90 | * The HWPoisoned page may be not in buddy system, and
|
| 91 | * page_count() is not 0.
|
| 92 |
|
| 93 | if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) && PageHWPoison(page))
|
| 94 | return false;
|
| 95 |
|
| 96 | |
| 97 | * We treat all PageOffline() pages as movable when offlining
|
| 98 | * to give drivers a chance to decrement their reference count
|
| 99 | * in MEM_GOING_OFFLINE in order to indicate that these pages
|
| 100 | * can be offlined as there are no direct references anymore.
|
| 101 | * For actually unmovable PageOffline() where the driver does
|
| 102 | * not support this, we will fail later when trying to actually
|
| 103 | * move these pages that still have a reference count > 0.
|
| 104 | * (false negatives in this function only)
|
| 105 |
|
| 106 | if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) && PageOffline(page))
|
| 107 | return false;
|
| 108 |
|
| 109 | if (PageLRU(page) || page_has_movable_ops(page))
|
| 110 | return false;
|
| 111 |
|
| 112 | |
| 113 | * If there are RECLAIMABLE pages, we need to check
|
| 114 | * it. But now, memory offline itself doesn't call
|
| 115 | * shrink_node_slabs() and it still to be fixed.
|
| 116 |
|
| 117 | return true;
|
| 118 | }
|
| 119 |
|
| 120 | |
| 121 | * This function checks whether the range [start_pfn, end_pfn) includes
|
| 122 | * unmovable pages or not. The range must fall into a single pageblock and
|
| 123 | * consequently belong to a single zone.
|
| 124 | *
|
| 125 | * PageLRU check without isolation or lru_lock could race so that
|
| 126 | * MIGRATE_MOVABLE block might include unmovable pages. Similarly, pages
|
| 127 | * with movable_ops can only be identified some time after they were
|
| 128 | * allocated. So you can't expect this function should be exact.
|
| 129 | *
|
| 130 | * Returns a page without holding a reference. If the caller wants to
|
| 131 | * dereference that page (e.g., dumping), it has to make sure that it
|
| 132 | * cannot get removed (e.g., via memory unplug) concurrently.
|
| 133 | *
|
| 134 |
|
| 135 | static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long end_pfn,
|
| 136 | enum pb_isolate_mode mode)
|
| 137 | {
|
| 138 | struct page *page = pfn_to_page(start_pfn);
|
| 139 | struct zone *zone = page_zone(page);
|
| 140 |
|
| 141 | VM_BUG_ON(pageblock_start_pfn(start_pfn) !=
|
| 142 | pageblock_start_pfn(end_pfn - 1));
|
| 143 |
|
| 144 | if (is_migrate_cma_page(page)) {
|
| 145 | |
| 146 | * CMA allocations (alloc_contig_range) really need to mark
|
| 147 | * isolate CMA pageblocks even when they are not movable in fact
|
| 148 | * so consider them movable here.
|
| 149 |
|
| 150 | if (mode == PB_ISOLATE_MODE_CMA_ALLOC)
|
| 151 | return NULL;
|
| 152 |
|
| 153 | return page;
|
| 154 | }
|
| 155 |
|
| 156 | while (start_pfn < end_pfn) {
|
| 157 | unsigned long step = 1;
|
| 158 |
|
| 159 | page = pfn_to_page(start_pfn);
|
| 160 | if (page_is_unmovable(zone, page, mode, &step))
|
| 161 | return page;
|
| 162 |
|
| 163 | start_pfn += step;
|
| 164 | }
|
| 165 | return NULL;
|
| 166 | }
|
| 167 |
|
| 168 | |
| 169 | * This function set pageblock migratetype to isolate if no unmovable page is
|
| 170 | * present in [start_pfn, end_pfn). The pageblock must intersect with
|
| 171 | * [start_pfn, end_pfn).
|
| 172 |
|
| 173 | static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode,
|
| 174 | unsigned long start_pfn, unsigned long end_pfn)
|
| 175 | {
|
| 176 | struct zone *zone = page_zone(page);
|
| 177 | struct page *unmovable;
|
| 178 | unsigned long check_unmovable_start, check_unmovable_end;
|
| 179 |
|
| 180 | if (PageUnaccepted(page))
|
| 181 | accept_page(page);
|
| 182 |
|
| 183 | scoped_guard(spinlock_irqsave, &zone->lock) {
|
| 184 | |
| 185 | * We assume the caller intended to SET migrate type to
|
| 186 | * isolate. If it is already set, then someone else must have
|
| 187 | * raced and set it before us.
|
| 188 |
|
| 189 | if (is_migrate_isolate_page(page))
|
| 190 | return -EBUSY;
|
| 191 |
|
| 192 | |
| 193 | * FIXME: Now, memory hotplug doesn't call shrink_slab() by
|
| 194 | * itself. We just check MOVABLE pages.
|
| 195 | *
|
| 196 | * Pass the intersection of [start_pfn, end_pfn) and the page's
|
| 197 | * pageblock to avoid redundant checks.
|
| 198 |
|
| 199 | check_unmovable_start = max(page_to_pfn(page), start_pfn);
|
| 200 | check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),
|
| 201 | end_pfn);
|
| 202 |
|
| 203 | unmovable = has_unmovable_pages(check_unmovable_start,
|
| 204 | check_unmovable_end, mode);
|
| 205 | if (!unmovable) {
|
| 206 | if (!pageblock_isolate_and_move_free_pages(zone, page))
|
| 207 | return -EBUSY;
|
| 208 | zone->nr_isolate_pageblock++;
|
| 209 | return 0;
|
| 210 | }
|
| 211 | }
|
| 212 | if (mode == PB_ISOLATE_MODE_MEM_OFFLINE) {
|
| 213 | |
| 214 | * printk() with zone->lock held will likely trigger a
|
| 215 | * lockdep splat, so defer it here.
|
| 216 |
|
| 217 | dump_page(unmovable, "unmovable page");
|
| 218 | }
|
| 219 |
|
| 220 | return -EBUSY;
|
| 221 | }
|
| 222 |
|
| 223 | static void unset_migratetype_isolate(struct page *page)
|
| 224 | {
|
| 225 | struct zone *zone;
|
| 226 | bool isolated_page = false;
|
| 227 | unsigned int order;
|
| 228 | struct page *buddy;
|
| 229 |
|
| 230 | zone = page_zone(page);
|
| 231 | guard(spinlock_irqsave)(&zone->lock);
|
| 232 | if (!is_migrate_isolate_page(page))
|
| 233 | return;
|
| 234 |
|
| 235 | |
| 236 | * Because freepage with more than pageblock_order on isolated
|
| 237 | * pageblock is restricted to merge due to freepage counting problem,
|
| 238 | * it is possible that there is free buddy page.
|
| 239 | * move_freepages_block() doesn't care of merge so we need other
|
| 240 | * approach in order to merge them. Isolation and free will make
|
| 241 | * these pages to be merged.
|
| 242 |
|
| 243 | if (PageBuddy(page)) {
|
| 244 | order = buddy_order(page);
|
| 245 | if (order >= pageblock_order && order < MAX_PAGE_ORDER) {
|
| 246 | buddy = find_buddy_page_pfn(page, page_to_pfn(page),
|
| 247 | order, NULL);
|
| 248 | if (buddy && !is_migrate_isolate_page(buddy)) {
|
| 249 | isolated_page = !!__isolate_free_page(page, order);
|
| 250 | |
| 251 | * Isolating a free page in an isolated pageblock
|
| 252 | * is expected to always work as watermarks don't
|
| 253 | * apply here.
|
| 254 |
|
| 255 | VM_WARN_ON(!isolated_page);
|
| 256 | }
|
| 257 | }
|
| 258 | }
|
| 259 |
|
| 260 | |
| 261 | * If we isolate freepage with more than pageblock_order, there
|
| 262 | * should be no freepage in the range, so we could avoid costly
|
| 263 | * pageblock scanning for freepage moving.
|
| 264 | *
|
| 265 | * We didn't actually touch any of the isolated pages, so place them
|
| 266 | * to the tail of the freelist. This is an optimization for memory
|
| 267 | * onlining - just onlined memory won't immediately be considered for
|
| 268 | * allocation.
|
| 269 |
|
| 270 | if (!isolated_page) {
|
| 271 | |
| 272 | * Isolating this block already succeeded, so this
|
| 273 | * should not fail on zone boundaries.
|
| 274 |
|
| 275 | WARN_ON_ONCE(!pageblock_unisolate_and_move_free_pages(zone, page));
|
| 276 | } else {
|
| 277 | clear_pageblock_isolate(page);
|
| 278 | __putback_isolated_page(page, order, get_pageblock_migratetype(page));
|
| 279 | }
|
| 280 | zone->nr_isolate_pageblock--;
|
| 281 | }
|
| 282 |
|
| 283 | static inline struct page *
|
| 284 | __first_valid_page(unsigned long pfn, unsigned long nr_pages)
|
| 285 | {
|
| 286 | int i;
|
| 287 |
|
| 288 | for (i = 0; i < nr_pages; i++) {
|
| 289 | struct page *page;
|
| 290 |
|
| 291 | page = pfn_to_online_page(pfn + i);
|
| 292 | if (!page)
|
| 293 | continue;
|
| 294 | return page;
|
| 295 | }
|
| 296 | return NULL;
|
| 297 | }
|
| 298 |
|
| 299 | |
| 300 | * isolate_single_pageblock() -- tries to isolate a pageblock that might be
|
| 301 | * within a free or in-use page.
|
| 302 | * @boundary_pfn: pageblock-aligned pfn that a page might cross
|
| 303 | * @mode: isolation mode
|
| 304 | * @isolate_before: isolate the pageblock before the boundary_pfn
|
| 305 | * @skip_isolation: the flag to skip the pageblock isolation in second
|
| 306 | * isolate_single_pageblock()
|
| 307 | *
|
| 308 | * Free and in-use pages can be as big as MAX_PAGE_ORDER and contain more than one
|
| 309 | * pageblock. When not all pageblocks within a page are isolated at the same
|
| 310 | * time, free page accounting can go wrong. For example, in the case of
|
| 311 | * MAX_PAGE_ORDER = pageblock_order + 1, a MAX_PAGE_ORDER page has two
|
| 312 | * pageblocks.
|
| 313 | * [ MAX_PAGE_ORDER ]
|
| 314 | * [ pageblock0 | pageblock1 ]
|
| 315 | * When either pageblock is isolated, if it is a free page, the page is not
|
| 316 | * split into separate migratetype lists, which is supposed to; if it is an
|
| 317 | * in-use page and freed later, __free_one_page() does not split the free page
|
| 318 | * either. The function handles this by splitting the free page or migrating
|
| 319 | * the in-use page then splitting the free page.
|
| 320 |
|
| 321 | static int isolate_single_pageblock(unsigned long boundary_pfn,
|
| 322 | enum pb_isolate_mode mode, bool isolate_before,
|
| 323 | bool skip_isolation)
|
| 324 | {
|
| 325 | unsigned long start_pfn;
|
| 326 | unsigned long isolate_pageblock;
|
| 327 | unsigned long pfn;
|
| 328 | struct zone *zone;
|
| 329 | int ret;
|
| 330 |
|
| 331 | VM_BUG_ON(!pageblock_aligned(boundary_pfn));
|
| 332 |
|
| 333 | if (isolate_before)
|
| 334 | isolate_pageblock = boundary_pfn - pageblock_nr_pages;
|
| 335 | else
|
| 336 | isolate_pageblock = boundary_pfn;
|
| 337 |
|
| 338 | |
| 339 | * scan at the beginning of MAX_ORDER_NR_PAGES aligned range to avoid
|
| 340 | * only isolating a subset of pageblocks from a bigger than pageblock
|
| 341 | * free or in-use page. Also make sure all to-be-isolated pageblocks
|
| 342 | * are within the same zone.
|
| 343 |
|
| 344 | zone = page_zone(pfn_to_page(isolate_pageblock));
|
| 345 | start_pfn = max(ALIGN_DOWN(isolate_pageblock, MAX_ORDER_NR_PAGES),
|
| 346 | zone->zone_start_pfn);
|
| 347 |
|
| 348 | if (skip_isolation) {
|
| 349 | VM_BUG_ON(!get_pageblock_isolate(pfn_to_page(isolate_pageblock)));
|
| 350 | } else {
|
| 351 | ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock),
|
| 352 | mode, isolate_pageblock,
|
| 353 | isolate_pageblock + pageblock_nr_pages);
|
| 354 |
|
| 355 | if (ret)
|
| 356 | return ret;
|
| 357 | }
|
| 358 |
|
| 359 | |
| 360 | * Bail out early when the to-be-isolated pageblock does not form
|
| 361 | * a free or in-use page across boundary_pfn:
|
| 362 | *
|
| 363 | * 1. isolate before boundary_pfn: the page after is not online
|
| 364 | * 2. isolate after boundary_pfn: the page before is not online
|
| 365 | *
|
| 366 | * This also ensures correctness. Without it, when isolate after
|
| 367 | * boundary_pfn and [start_pfn, boundary_pfn) are not online,
|
| 368 | * __first_valid_page() will return unexpected NULL in the for loop
|
| 369 | * below.
|
| 370 |
|
| 371 | if (isolate_before) {
|
| 372 | if (!pfn_to_online_page(boundary_pfn))
|
| 373 | return 0;
|
| 374 | } else {
|
| 375 | if (!pfn_to_online_page(boundary_pfn - 1))
|
| 376 | return 0;
|
| 377 | }
|
| 378 |
|
| 379 | for (pfn = start_pfn; pfn < boundary_pfn;) {
|
| 380 | struct page *page = __first_valid_page(pfn, boundary_pfn - pfn);
|
| 381 |
|
| 382 | VM_BUG_ON(!page);
|
| 383 | pfn = page_to_pfn(page);
|
| 384 |
|
| 385 | if (PageUnaccepted(page)) {
|
| 386 | pfn += MAX_ORDER_NR_PAGES;
|
| 387 | continue;
|
| 388 | }
|
| 389 |
|
| 390 | if (PageBuddy(page)) {
|
| 391 | int order = buddy_order(page);
|
| 392 |
|
| 393 |
|
| 394 | VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn);
|
| 395 |
|
| 396 | pfn += 1UL << order;
|
| 397 | continue;
|
| 398 | }
|
| 399 |
|
| 400 | |
| 401 | * If a compound page is straddling our block, attempt
|
| 402 | * to migrate it out of the way.
|
| 403 | *
|
| 404 | * We don't have to worry about this creating a large
|
| 405 | * free page that straddles into our block: gigantic
|
| 406 | * pages are freed as order-0 chunks, and LRU pages
|
| 407 | * (currently) do not exceed pageblock_order.
|
| 408 | *
|
| 409 | * The block of interest has already been marked
|
| 410 | * MIGRATE_ISOLATE above, so when migration is done it
|
| 411 | * will free its pages onto the correct freelists.
|
| 412 |
|
| 413 | if (PageCompound(page)) {
|
| 414 | struct page *head = compound_head(page);
|
| 415 | unsigned long head_pfn = page_to_pfn(head);
|
| 416 | unsigned long nr_pages = compound_nr(head);
|
| 417 |
|
| 418 | if (head_pfn + nr_pages <= boundary_pfn ||
|
| 419 | PageHuge(page)) {
|
| 420 | pfn = head_pfn + nr_pages;
|
| 421 | continue;
|
| 422 | }
|
| 423 |
|
| 424 | |
| 425 | * These pages are movable too, but they're
|
| 426 | * not expected to exceed pageblock_order.
|
| 427 | *
|
| 428 | * Let us know when they do, so we can add
|
| 429 | * proper free and split handling for them.
|
| 430 |
|
| 431 | VM_WARN_ON_ONCE_PAGE(PageLRU(page), page);
|
| 432 | VM_WARN_ON_ONCE_PAGE(page_has_movable_ops(page), page);
|
| 433 |
|
| 434 | goto failed;
|
| 435 | }
|
| 436 |
|
| 437 | pfn++;
|
| 438 | }
|
| 439 | return 0;
|
| 440 | failed:
|
| 441 |
|
| 442 | if (!skip_isolation)
|
| 443 | unset_migratetype_isolate(pfn_to_page(isolate_pageblock));
|
| 444 | return -EBUSY;
|
| 445 | }
|
| 446 |
|
| 447 | |
| 448 | * start_isolate_page_range() - mark page range MIGRATE_ISOLATE
|
| 449 | * @start_pfn: The first PFN of the range to be isolated.
|
| 450 | * @end_pfn: The last PFN of the range to be isolated.
|
| 451 | * @mode: isolation mode
|
| 452 | *
|
| 453 | * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
|
| 454 | * the range will never be allocated. Any free pages and pages freed in the
|
| 455 | * future will not be allocated again. If specified range includes migrate types
|
| 456 | * other than MOVABLE or CMA, this will fail with -EBUSY. For isolating all
|
| 457 | * pages in the range finally, the caller have to free all pages in the range.
|
| 458 | * test_page_isolated() can be used for test it.
|
| 459 | *
|
| 460 | * The function first tries to isolate the pageblocks at the beginning and end
|
| 461 | * of the range, since there might be pages across the range boundaries.
|
| 462 | * Afterwards, it isolates the rest of the range.
|
| 463 | *
|
| 464 | * There is no high level synchronization mechanism that prevents two threads
|
| 465 | * from trying to isolate overlapping ranges. If this happens, one thread
|
| 466 | * will notice pageblocks in the overlapping range already set to isolate.
|
| 467 | * This happens in set_migratetype_isolate, and set_migratetype_isolate
|
| 468 | * returns an error. We then clean up by restoring the migration type on
|
| 469 | * pageblocks we may have modified and return -EBUSY to caller. This
|
| 470 | * prevents two threads from simultaneously working on overlapping ranges.
|
| 471 | *
|
| 472 | * Please note that there is no strong synchronization with the page allocator
|
| 473 | * either. Pages might be freed while their page blocks are marked ISOLATED.
|
| 474 | * A call to drain_all_pages() after isolation can flush most of them. However
|
| 475 | * in some cases pages might still end up on pcp lists and that would allow
|
| 476 | * for their allocation even when they are in fact isolated already. Depending
|
| 477 | * on how strong of a guarantee the caller needs, zone_pcp_disable/enable()
|
| 478 | * might be used to flush and disable pcplist before isolation and enable after
|
| 479 | * unisolation.
|
| 480 | *
|
| 481 | * Return: 0 on success and -EBUSY if any part of range cannot be isolated.
|
| 482 |
|
| 483 | int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
|
| 484 | enum pb_isolate_mode mode)
|
| 485 | {
|
| 486 | unsigned long pfn;
|
| 487 | struct page *page;
|
| 488 |
|
| 489 | unsigned long isolate_start = pageblock_start_pfn(start_pfn);
|
| 490 | unsigned long isolate_end = pageblock_align(end_pfn);
|
| 491 | int ret;
|
| 492 | bool skip_isolation = false;
|
| 493 |
|
| 494 |
|
| 495 | ret = isolate_single_pageblock(isolate_start, mode, false,
|
| 496 | skip_isolation);
|
| 497 | if (ret)
|
| 498 | return ret;
|
| 499 |
|
| 500 | if (isolate_start == isolate_end - pageblock_nr_pages)
|
| 501 | skip_isolation = true;
|
| 502 |
|
| 503 |
|
| 504 | ret = isolate_single_pageblock(isolate_end, mode, true, skip_isolation);
|
| 505 | if (ret) {
|
| 506 | unset_migratetype_isolate(pfn_to_page(isolate_start));
|
| 507 | return ret;
|
| 508 | }
|
| 509 |
|
| 510 |
|
| 511 | for (pfn = isolate_start + pageblock_nr_pages;
|
| 512 | pfn < isolate_end - pageblock_nr_pages;
|
| 513 | pfn += pageblock_nr_pages) {
|
| 514 | page = __first_valid_page(pfn, pageblock_nr_pages);
|
| 515 | if (page && set_migratetype_isolate(page, mode, start_pfn,
|
| 516 | end_pfn)) {
|
| 517 | undo_isolate_page_range(isolate_start, pfn);
|
| 518 | unset_migratetype_isolate(
|
| 519 | pfn_to_page(isolate_end - pageblock_nr_pages));
|
| 520 | return -EBUSY;
|
| 521 | }
|
| 522 | }
|
| 523 | return 0;
|
| 524 | }
|
| 525 |
|
| 526 | |
| 527 | * undo_isolate_page_range - undo effects of start_isolate_page_range()
|
| 528 | * @start_pfn: The first PFN of the isolated range
|
| 529 | * @end_pfn: The last PFN of the isolated range
|
| 530 | *
|
| 531 | * This finds and unsets every MIGRATE_ISOLATE page block in the given range
|
| 532 |
|
| 533 | void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn)
|
| 534 | {
|
| 535 | unsigned long pfn;
|
| 536 | struct page *page;
|
| 537 | unsigned long isolate_start = pageblock_start_pfn(start_pfn);
|
| 538 | unsigned long isolate_end = pageblock_align(end_pfn);
|
| 539 |
|
| 540 | for (pfn = isolate_start;
|
| 541 | pfn < isolate_end;
|
| 542 | pfn += pageblock_nr_pages) {
|
| 543 | page = __first_valid_page(pfn, pageblock_nr_pages);
|
| 544 | if (!page || !is_migrate_isolate_page(page))
|
| 545 | continue;
|
| 546 | unset_migratetype_isolate(page);
|
| 547 | }
|
| 548 | }
|
| 549 | |
| 550 | * Test all pages in the range is free(means isolated) or not.
|
| 551 | * all pages in [start_pfn...end_pfn) must be in the same zone.
|
| 552 | * zone->lock must be held before call this.
|
| 553 | *
|
| 554 | * Returns the last tested pfn.
|
| 555 |
|
| 556 | static unsigned long
|
| 557 | __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
|
| 558 | enum pb_isolate_mode mode)
|
| 559 | {
|
| 560 | struct page *page;
|
| 561 |
|
| 562 | while (pfn < end_pfn) {
|
| 563 | page = pfn_to_page(pfn);
|
| 564 | if (PageBuddy(page))
|
| 565 | |
| 566 | * If the page is on a free list, it has to be on
|
| 567 | * the correct MIGRATE_ISOLATE freelist. There is no
|
| 568 | * simple way to verify that as VM_BUG_ON(), though.
|
| 569 |
|
| 570 | pfn += 1 << buddy_order(page);
|
| 571 | else if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) &&
|
| 572 | PageHWPoison(page))
|
| 573 |
|
| 574 | pfn++;
|
| 575 | else if ((mode == PB_ISOLATE_MODE_MEM_OFFLINE) &&
|
| 576 | PageOffline(page) && !page_count(page))
|
| 577 | |
| 578 | * The responsible driver agreed to skip PageOffline()
|
| 579 | * pages when offlining memory by dropping its
|
| 580 | * reference in MEM_GOING_OFFLINE.
|
| 581 |
|
| 582 | pfn++;
|
| 583 | else
|
| 584 | break;
|
| 585 | }
|
| 586 |
|
| 587 | return pfn;
|
| 588 | }
|
| 589 |
|
| 590 | |
| 591 | * test_pages_isolated - check if pageblocks in range are isolated
|
| 592 | * @start_pfn: The first PFN of the isolated range
|
| 593 | * @end_pfn: The first PFN *after* the isolated range
|
| 594 | * @mode: Testing mode
|
| 595 | *
|
| 596 | * This tests if all in the specified range are free.
|
| 597 | *
|
| 598 | * If %PB_ISOLATE_MODE_MEM_OFFLINE specified in @mode, it will consider
|
| 599 | * poisoned and offlined pages free as well.
|
| 600 | *
|
| 601 | * Caller must ensure the requested range doesn't span zones.
|
| 602 | *
|
| 603 | * Returns 0 if true, -EBUSY if one or more pages are in use.
|
| 604 |
|
| 605 | int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
|
| 606 | enum pb_isolate_mode mode)
|
| 607 | {
|
| 608 | unsigned long pfn, flags;
|
| 609 | struct page *page;
|
| 610 | struct zone *zone;
|
| 611 | int ret;
|
| 612 |
|
| 613 | |
| 614 | * Due to the deferred freeing of hugetlb folios, the hugepage folios may
|
| 615 | * not immediately release to the buddy system. This can cause PageBuddy()
|
| 616 | * to fail in __test_page_isolated_in_pageblock(). To ensure that the
|
| 617 | * hugetlb folios are properly released back to the buddy system, we
|
| 618 | * invoke the wait_for_freed_hugetlb_folios() function to wait for the
|
| 619 | * release to complete.
|
| 620 |
|
| 621 | wait_for_freed_hugetlb_folios();
|
| 622 |
|
| 623 | |
| 624 | * Note: pageblock_nr_pages != MAX_PAGE_ORDER. Then, chunks of free
|
| 625 | * pages are not aligned to pageblock_nr_pages.
|
| 626 | * Then we just check migratetype first.
|
| 627 |
|
| 628 | for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
|
| 629 | page = __first_valid_page(pfn, pageblock_nr_pages);
|
| 630 | if (page && !is_migrate_isolate_page(page))
|
| 631 | break;
|
| 632 | }
|
| 633 | page = __first_valid_page(start_pfn, end_pfn - start_pfn);
|
| 634 | if ((pfn < end_pfn) || !page) {
|
| 635 | ret = -EBUSY;
|
| 636 | goto out;
|
| 637 | }
|
| 638 |
|
| 639 |
|
| 640 | zone = page_zone(page);
|
| 641 | spin_lock_irqsave(&zone->lock, flags);
|
| 642 | pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn, mode);
|
| 643 | spin_unlock_irqrestore(&zone->lock, flags);
|
| 644 |
|
| 645 | ret = pfn < end_pfn ? -EBUSY : 0;
|
| 646 |
|
| 647 | out:
|
| 648 | trace_test_pages_isolated(start_pfn, end_pfn, pfn);
|
| 649 |
|
| 650 | return ret;
|
| 651 | }
|