| 1 | |
| 2 | |
| 3 | * linux/mm/compaction.c
|
| 4 | *
|
| 5 | * Memory compaction for the reduction of external fragmentation. Note that
|
| 6 | * this heavily depends upon page migration to do all the real heavy
|
| 7 | * lifting
|
| 8 | *
|
| 9 | * Copyright IBM Corp. 2007-2010 Mel Gorman <[email protected]>
|
| 10 |
|
| 11 | #include <linux/cpu.h>
|
| 12 | #include <linux/swap.h>
|
| 13 | #include <linux/migrate.h>
|
| 14 | #include <linux/compaction.h>
|
| 15 | #include <linux/mm_inline.h>
|
| 16 | #include <linux/sched/signal.h>
|
| 17 | #include <linux/backing-dev.h>
|
| 18 | #include <linux/sysctl.h>
|
| 19 | #include <linux/sysfs.h>
|
| 20 | #include <linux/page-isolation.h>
|
| 21 | #include <linux/kasan.h>
|
| 22 | #include <linux/kthread.h>
|
| 23 | #include <linux/freezer.h>
|
| 24 | #include <linux/page_owner.h>
|
| 25 | #include <linux/psi.h>
|
| 26 | #include <linux/cpuset.h>
|
| 27 | #include "page_alloc.h"
|
| 28 | #include "internal.h"
|
| 29 |
|
| 30 | #ifdef CONFIG_COMPACTION
|
| 31 | |
| 32 | * Fragmentation score check interval for proactive compaction purposes.
|
| 33 |
|
| 34 | #define HPAGE_FRAG_CHECK_INTERVAL_MSEC (500)
|
| 35 |
|
| 36 | static inline void count_compact_event(enum vm_event_item item)
|
| 37 | {
|
| 38 | count_vm_event(item);
|
| 39 | }
|
| 40 |
|
| 41 | static inline void count_compact_events(enum vm_event_item item, long delta)
|
| 42 | {
|
| 43 | count_vm_events(item, delta);
|
| 44 | }
|
| 45 |
|
| 46 | |
| 47 | * order == -1 is expected when compacting proactively via
|
| 48 | * 1. /proc/sys/vm/compact_memory
|
| 49 | * 2. /sys/devices/system/node/nodex/compact
|
| 50 | * 3. /proc/sys/vm/compaction_proactiveness
|
| 51 |
|
| 52 | static inline bool is_via_compact_memory(int order)
|
| 53 | {
|
| 54 | return order == -1;
|
| 55 | }
|
| 56 |
|
| 57 | #else
|
| 58 | #define count_compact_event(item) do { } while (0)
|
| 59 | #define count_compact_events(item, delta) do { } while (0)
|
| 60 | static inline bool is_via_compact_memory(int order) { return false; }
|
| 61 | #endif
|
| 62 |
|
| 63 | #if defined CONFIG_COMPACTION || defined CONFIG_CMA
|
| 64 |
|
| 65 | #define CREATE_TRACE_POINTS
|
| 66 | #include <trace/events/compaction.h>
|
| 67 |
|
| 68 | #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
|
| 69 | #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
|
| 70 |
|
| 71 | |
| 72 | * Page order with-respect-to which proactive compaction
|
| 73 | * calculates external fragmentation, which is used as
|
| 74 | * the "fragmentation score" of a node/zone.
|
| 75 |
|
| 76 | #if defined CONFIG_TRANSPARENT_HUGEPAGE
|
| 77 | #define COMPACTION_HPAGE_ORDER HPAGE_PMD_ORDER
|
| 78 | #elif defined CONFIG_HUGETLBFS
|
| 79 | #define COMPACTION_HPAGE_ORDER HUGETLB_PAGE_ORDER
|
| 80 | #else
|
| 81 | #define COMPACTION_HPAGE_ORDER (PMD_SHIFT - PAGE_SHIFT)
|
| 82 | #endif
|
| 83 |
|
| 84 | static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags)
|
| 85 | {
|
| 86 | post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT);
|
| 87 | set_page_refcounted(page);
|
| 88 | return page;
|
| 89 | }
|
| 90 | #define mark_allocated(...) alloc_hooks(mark_allocated_noprof(__VA_ARGS__))
|
| 91 |
|
| 92 | static unsigned long release_free_list(struct list_head *freepages)
|
| 93 | {
|
| 94 | int order;
|
| 95 | unsigned long high_pfn = 0;
|
| 96 |
|
| 97 | for (order = 0; order < NR_PAGE_ORDERS; order++) {
|
| 98 | struct page *page, *next;
|
| 99 |
|
| 100 | list_for_each_entry_safe(page, next, &freepages[order], lru) {
|
| 101 | unsigned long pfn = page_to_pfn(page);
|
| 102 |
|
| 103 | list_del(&page->lru);
|
| 104 | |
| 105 | * Convert free pages into post allocation pages, so
|
| 106 | * that we can free them via __free_page.
|
| 107 |
|
| 108 | mark_allocated(page, order, __GFP_MOVABLE);
|
| 109 | __free_pages(page, order);
|
| 110 | if (pfn > high_pfn)
|
| 111 | high_pfn = pfn;
|
| 112 | }
|
| 113 | }
|
| 114 | return high_pfn;
|
| 115 | }
|
| 116 |
|
| 117 | #ifdef CONFIG_COMPACTION
|
| 118 |
|
| 119 |
|
| 120 | #define COMPACT_MAX_DEFER_SHIFT 6
|
| 121 |
|
| 122 | |
| 123 | * Compaction is deferred when compaction fails to result in a page
|
| 124 | * allocation success. 1 << compact_defer_shift, compactions are skipped up
|
| 125 | * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
|
| 126 |
|
| 127 | static void defer_compaction(struct zone *zone, int order)
|
| 128 | {
|
| 129 | zone->compact_considered = 0;
|
| 130 | zone->compact_defer_shift++;
|
| 131 |
|
| 132 | if (order < zone->compact_order_failed)
|
| 133 | zone->compact_order_failed = order;
|
| 134 |
|
| 135 | if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
|
| 136 | zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
|
| 137 |
|
| 138 | trace_mm_compaction_defer_compaction(zone, order);
|
| 139 | }
|
| 140 |
|
| 141 |
|
| 142 | static bool compaction_deferred(struct zone *zone, int order)
|
| 143 | {
|
| 144 | unsigned long defer_limit = 1UL << zone->compact_defer_shift;
|
| 145 |
|
| 146 | if (order < zone->compact_order_failed)
|
| 147 | return false;
|
| 148 |
|
| 149 |
|
| 150 | if (++zone->compact_considered >= defer_limit) {
|
| 151 | zone->compact_considered = defer_limit;
|
| 152 | return false;
|
| 153 | }
|
| 154 |
|
| 155 | trace_mm_compaction_deferred(zone, order);
|
| 156 |
|
| 157 | return true;
|
| 158 | }
|
| 159 |
|
| 160 | |
| 161 | * Update defer tracking counters after successful compaction of given order,
|
| 162 | * which means an allocation either succeeded (alloc_success == true) or is
|
| 163 | * expected to succeed.
|
| 164 |
|
| 165 | void compaction_defer_reset(struct zone *zone, int order,
|
| 166 | bool alloc_success)
|
| 167 | {
|
| 168 | if (alloc_success) {
|
| 169 | zone->compact_considered = 0;
|
| 170 | zone->compact_defer_shift = 0;
|
| 171 | }
|
| 172 | if (order >= zone->compact_order_failed)
|
| 173 | zone->compact_order_failed = order + 1;
|
| 174 |
|
| 175 | trace_mm_compaction_defer_reset(zone, order);
|
| 176 | }
|
| 177 |
|
| 178 |
|
| 179 | static bool compaction_restarting(struct zone *zone, int order)
|
| 180 | {
|
| 181 | if (order < zone->compact_order_failed)
|
| 182 | return false;
|
| 183 |
|
| 184 | return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
|
| 185 | zone->compact_considered >= 1UL << zone->compact_defer_shift;
|
| 186 | }
|
| 187 |
|
| 188 |
|
| 189 | static inline bool isolation_suitable(struct compact_control *cc,
|
| 190 | struct page *page)
|
| 191 | {
|
| 192 | if (cc->ignore_skip_hint)
|
| 193 | return true;
|
| 194 |
|
| 195 | return !get_pageblock_skip(page);
|
| 196 | }
|
| 197 |
|
| 198 | static void reset_cached_positions(struct zone *zone)
|
| 199 | {
|
| 200 | zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
|
| 201 | zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
|
| 202 | zone->compact_cached_free_pfn =
|
| 203 | pageblock_start_pfn(zone_end_pfn(zone) - 1);
|
| 204 | }
|
| 205 |
|
| 206 | #ifdef CONFIG_SPARSEMEM
|
| 207 | |
| 208 | * If the PFN falls into an offline section, return the start PFN of the
|
| 209 | * next online section. If the PFN falls into an online section or if
|
| 210 | * there is no next online section, return 0.
|
| 211 |
|
| 212 | static unsigned long skip_offline_sections(unsigned long start_pfn)
|
| 213 | {
|
| 214 | unsigned long start_nr = pfn_to_section_nr(start_pfn);
|
| 215 |
|
| 216 | if (online_section_nr(start_nr))
|
| 217 | return 0;
|
| 218 |
|
| 219 | while (++start_nr <= __highest_present_section_nr) {
|
| 220 | if (online_section_nr(start_nr))
|
| 221 | return section_nr_to_pfn(start_nr);
|
| 222 | }
|
| 223 |
|
| 224 | return 0;
|
| 225 | }
|
| 226 |
|
| 227 | |
| 228 | * If the PFN falls into an offline section, return the end PFN of the
|
| 229 | * next online section in reverse. If the PFN falls into an online section
|
| 230 | * or if there is no next online section in reverse, return 0.
|
| 231 |
|
| 232 | static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
|
| 233 | {
|
| 234 | unsigned long start_nr = pfn_to_section_nr(start_pfn);
|
| 235 |
|
| 236 | if (!start_nr || online_section_nr(start_nr))
|
| 237 | return 0;
|
| 238 |
|
| 239 | while (start_nr-- > 0) {
|
| 240 | if (online_section_nr(start_nr))
|
| 241 | return section_nr_to_pfn(start_nr) + PAGES_PER_SECTION;
|
| 242 | }
|
| 243 |
|
| 244 | return 0;
|
| 245 | }
|
| 246 | #else
|
| 247 | static unsigned long skip_offline_sections(unsigned long start_pfn)
|
| 248 | {
|
| 249 | return 0;
|
| 250 | }
|
| 251 |
|
| 252 | static unsigned long skip_offline_sections_reverse(unsigned long start_pfn)
|
| 253 | {
|
| 254 | return 0;
|
| 255 | }
|
| 256 | #endif
|
| 257 |
|
| 258 | |
| 259 | * Compound pages of >= pageblock_order should consistently be skipped until
|
| 260 | * released. It is always pointless to compact pages of such order (if they are
|
| 261 | * migratable), and the pageblocks they occupy cannot contain any free pages.
|
| 262 |
|
| 263 | static bool pageblock_skip_persistent(struct page *page)
|
| 264 | {
|
| 265 | if (!PageCompound(page))
|
| 266 | return false;
|
| 267 |
|
| 268 | page = compound_head(page);
|
| 269 |
|
| 270 | if (compound_order(page) >= pageblock_order)
|
| 271 | return true;
|
| 272 |
|
| 273 | return false;
|
| 274 | }
|
| 275 |
|
| 276 | static bool
|
| 277 | __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,
|
| 278 | bool check_target)
|
| 279 | {
|
| 280 | struct page *page = pfn_to_online_page(pfn);
|
| 281 | struct page *block_page;
|
| 282 | struct page *end_page;
|
| 283 | unsigned long block_pfn;
|
| 284 |
|
| 285 | if (!page)
|
| 286 | return false;
|
| 287 | if (zone != page_zone(page))
|
| 288 | return false;
|
| 289 | if (pageblock_skip_persistent(page))
|
| 290 | return false;
|
| 291 |
|
| 292 | |
| 293 | * If skip is already cleared do no further checking once the
|
| 294 | * restart points have been set.
|
| 295 |
|
| 296 | if (check_source && check_target && !get_pageblock_skip(page))
|
| 297 | return true;
|
| 298 |
|
| 299 | |
| 300 | * If clearing skip for the target scanner, do not select a
|
| 301 | * non-movable pageblock as the starting point.
|
| 302 |
|
| 303 | if (!check_source && check_target &&
|
| 304 | get_pageblock_migratetype(page) != MIGRATE_MOVABLE)
|
| 305 | return false;
|
| 306 |
|
| 307 |
|
| 308 | block_pfn = pageblock_start_pfn(pfn);
|
| 309 | block_pfn = max(block_pfn, zone->zone_start_pfn);
|
| 310 | block_page = pfn_to_online_page(block_pfn);
|
| 311 | if (block_page) {
|
| 312 | page = block_page;
|
| 313 | pfn = block_pfn;
|
| 314 | }
|
| 315 |
|
| 316 |
|
| 317 | block_pfn = pageblock_end_pfn(pfn) - 1;
|
| 318 | block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);
|
| 319 | end_page = pfn_to_online_page(block_pfn);
|
| 320 | if (!end_page)
|
| 321 | return false;
|
| 322 |
|
| 323 | |
| 324 | * Only clear the hint if a sample indicates there is either a
|
| 325 | * free page or an LRU page in the block. One or other condition
|
| 326 | * is necessary for the block to be a migration source/target.
|
| 327 |
|
| 328 | do {
|
| 329 | if (check_source && PageLRU(page)) {
|
| 330 | clear_pageblock_skip(page);
|
| 331 | return true;
|
| 332 | }
|
| 333 |
|
| 334 | if (check_target && PageBuddy(page)) {
|
| 335 | clear_pageblock_skip(page);
|
| 336 | return true;
|
| 337 | }
|
| 338 |
|
| 339 | page += (1 << PAGE_ALLOC_COSTLY_ORDER);
|
| 340 | } while (page <= end_page);
|
| 341 |
|
| 342 | return false;
|
| 343 | }
|
| 344 |
|
| 345 | |
| 346 | * This function is called to clear all cached information on pageblocks that
|
| 347 | * should be skipped for page isolation when the migrate and free page scanner
|
| 348 | * meet.
|
| 349 |
|
| 350 | static void __reset_isolation_suitable(struct zone *zone)
|
| 351 | {
|
| 352 | unsigned long migrate_pfn = zone->zone_start_pfn;
|
| 353 | unsigned long free_pfn = zone_end_pfn(zone) - 1;
|
| 354 | unsigned long reset_migrate = free_pfn;
|
| 355 | unsigned long reset_free = migrate_pfn;
|
| 356 | bool source_set = false;
|
| 357 | bool free_set = false;
|
| 358 |
|
| 359 |
|
| 360 | if (!zone->compact_blockskip_flush)
|
| 361 | return;
|
| 362 |
|
| 363 | zone->compact_blockskip_flush = false;
|
| 364 |
|
| 365 | |
| 366 | * Walk the zone and update pageblock skip information. Source looks
|
| 367 | * for PageLRU while target looks for PageBuddy. When the scanner
|
| 368 | * is found, both PageBuddy and PageLRU are checked as the pageblock
|
| 369 | * is suitable as both source and target.
|
| 370 |
|
| 371 | for (; migrate_pfn < free_pfn; migrate_pfn += pageblock_nr_pages,
|
| 372 | free_pfn -= pageblock_nr_pages) {
|
| 373 | cond_resched();
|
| 374 |
|
| 375 |
|
| 376 | if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&
|
| 377 | migrate_pfn < reset_migrate) {
|
| 378 | source_set = true;
|
| 379 | reset_migrate = migrate_pfn;
|
| 380 | zone->compact_init_migrate_pfn = reset_migrate;
|
| 381 | zone->compact_cached_migrate_pfn[0] = reset_migrate;
|
| 382 | zone->compact_cached_migrate_pfn[1] = reset_migrate;
|
| 383 | }
|
| 384 |
|
| 385 |
|
| 386 | if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&
|
| 387 | free_pfn > reset_free) {
|
| 388 | free_set = true;
|
| 389 | reset_free = free_pfn;
|
| 390 | zone->compact_init_free_pfn = reset_free;
|
| 391 | zone->compact_cached_free_pfn = reset_free;
|
| 392 | }
|
| 393 | }
|
| 394 |
|
| 395 |
|
| 396 | if (reset_migrate >= reset_free) {
|
| 397 | zone->compact_cached_migrate_pfn[0] = migrate_pfn;
|
| 398 | zone->compact_cached_migrate_pfn[1] = migrate_pfn;
|
| 399 | zone->compact_cached_free_pfn = free_pfn;
|
| 400 | }
|
| 401 | }
|
| 402 |
|
| 403 | void reset_isolation_suitable(pg_data_t *pgdat)
|
| 404 | {
|
| 405 | int zoneid;
|
| 406 |
|
| 407 | for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
|
| 408 | struct zone *zone = &pgdat->node_zones[zoneid];
|
| 409 | if (!populated_zone(zone))
|
| 410 | continue;
|
| 411 |
|
| 412 | __reset_isolation_suitable(zone);
|
| 413 | }
|
| 414 | }
|
| 415 |
|
| 416 | |
| 417 | * Sets the pageblock skip bit if it was clear. Note that this is a hint as
|
| 418 | * locks are not required for read/writers. Returns true if it was already set.
|
| 419 |
|
| 420 | static bool test_and_set_skip(struct compact_control *cc, struct page *page)
|
| 421 | {
|
| 422 | bool skip;
|
| 423 |
|
| 424 |
|
| 425 | if (cc->ignore_skip_hint)
|
| 426 | return false;
|
| 427 |
|
| 428 | skip = get_pageblock_skip(page);
|
| 429 | if (!skip && !cc->no_set_skip_hint)
|
| 430 | set_pageblock_skip(page);
|
| 431 |
|
| 432 | return skip;
|
| 433 | }
|
| 434 |
|
| 435 | static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
|
| 436 | {
|
| 437 | struct zone *zone = cc->zone;
|
| 438 |
|
| 439 |
|
| 440 | if (cc->no_set_skip_hint)
|
| 441 | return;
|
| 442 |
|
| 443 | pfn = pageblock_end_pfn(pfn);
|
| 444 |
|
| 445 |
|
| 446 | if (pfn > zone->compact_cached_migrate_pfn[0])
|
| 447 | zone->compact_cached_migrate_pfn[0] = pfn;
|
| 448 | if (cc->mode != MIGRATE_ASYNC &&
|
| 449 | pfn > zone->compact_cached_migrate_pfn[1])
|
| 450 | zone->compact_cached_migrate_pfn[1] = pfn;
|
| 451 | }
|
| 452 |
|
| 453 | |
| 454 | * If no pages were isolated then mark this pageblock to be skipped in the
|
| 455 | * future. The information is later cleared by __reset_isolation_suitable().
|
| 456 |
|
| 457 | static void update_pageblock_skip(struct compact_control *cc,
|
| 458 | struct page *page, unsigned long pfn)
|
| 459 | {
|
| 460 | struct zone *zone = cc->zone;
|
| 461 |
|
| 462 | if (cc->no_set_skip_hint)
|
| 463 | return;
|
| 464 |
|
| 465 | set_pageblock_skip(page);
|
| 466 |
|
| 467 | if (pfn < zone->compact_cached_free_pfn)
|
| 468 | zone->compact_cached_free_pfn = pfn;
|
| 469 | }
|
| 470 | #else
|
| 471 | static inline bool isolation_suitable(struct compact_control *cc,
|
| 472 | struct page *page)
|
| 473 | {
|
| 474 | return true;
|
| 475 | }
|
| 476 |
|
| 477 | static inline bool pageblock_skip_persistent(struct page *page)
|
| 478 | {
|
| 479 | return false;
|
| 480 | }
|
| 481 |
|
| 482 | static inline void update_pageblock_skip(struct compact_control *cc,
|
| 483 | struct page *page, unsigned long pfn)
|
| 484 | {
|
| 485 | }
|
| 486 |
|
| 487 | static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
|
| 488 | {
|
| 489 | }
|
| 490 |
|
| 491 | static bool test_and_set_skip(struct compact_control *cc, struct page *page)
|
| 492 | {
|
| 493 | return false;
|
| 494 | }
|
| 495 | #endif
|
| 496 |
|
| 497 | |
| 498 | * Compaction requires the taking of some coarse locks that are potentially
|
| 499 | * very heavily contended. For async compaction, trylock and record if the
|
| 500 | * lock is contended. The lock will still be acquired but compaction will
|
| 501 | * abort when the current block is finished regardless of success rate.
|
| 502 | * Sync compaction acquires the lock.
|
| 503 | *
|
| 504 | * Always returns true which makes it easier to track lock state in callers.
|
| 505 |
|
| 506 | static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
|
| 507 | struct compact_control *cc)
|
| 508 | __acquires(lock)
|
| 509 | {
|
| 510 |
|
| 511 | if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
|
| 512 | if (spin_trylock_irqsave(lock, *flags))
|
| 513 | return true;
|
| 514 |
|
| 515 | cc->contended = true;
|
| 516 | }
|
| 517 |
|
| 518 | spin_lock_irqsave(lock, *flags);
|
| 519 | return true;
|
| 520 | }
|
| 521 |
|
| 522 | static struct lruvec *
|
| 523 | compact_folio_lruvec_lock_irqsave(struct folio *folio, unsigned long *flags,
|
| 524 | struct compact_control *cc)
|
| 525 | {
|
| 526 | struct lruvec *lruvec;
|
| 527 |
|
| 528 | rcu_read_lock();
|
| 529 | retry:
|
| 530 | lruvec = folio_lruvec(folio);
|
| 531 | compact_lock_irqsave(&lruvec->lru_lock, flags, cc);
|
| 532 | if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
|
| 533 | spin_unlock_irqrestore(&lruvec->lru_lock, *flags);
|
| 534 | goto retry;
|
| 535 | }
|
| 536 |
|
| 537 | return lruvec;
|
| 538 | }
|
| 539 |
|
| 540 | |
| 541 | * Compaction requires the taking of some coarse locks that are potentially
|
| 542 | * very heavily contended. The lock should be periodically unlocked to avoid
|
| 543 | * having disabled IRQs for a long time, even when there is nobody waiting on
|
| 544 | * the lock. It might also be that allowing the IRQs will result in
|
| 545 | * need_resched() becoming true. If scheduling is needed, compaction schedules.
|
| 546 | * Either compaction type will also abort if a fatal signal is pending.
|
| 547 | * In either case if the lock was locked, it is dropped and not regained.
|
| 548 | *
|
| 549 | * Returns true if compaction should abort due to fatal signal pending.
|
| 550 | * Returns false when compaction can continue.
|
| 551 |
|
| 552 | static bool compact_unlock_should_abort(spinlock_t *lock,
|
| 553 | unsigned long flags, bool *locked, struct compact_control *cc)
|
| 554 | {
|
| 555 | if (*locked) {
|
| 556 | spin_unlock_irqrestore(lock, flags);
|
| 557 | *locked = false;
|
| 558 | }
|
| 559 |
|
| 560 | if (fatal_signal_pending(current)) {
|
| 561 | cc->contended = true;
|
| 562 | return true;
|
| 563 | }
|
| 564 |
|
| 565 | cond_resched();
|
| 566 |
|
| 567 | return false;
|
| 568 | }
|
| 569 |
|
| 570 | |
| 571 | * Isolate free pages onto a private freelist. If @strict is true, will abort
|
| 572 | * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
|
| 573 | * (even though it may still end up isolating some pages).
|
| 574 |
|
| 575 | static unsigned long isolate_freepages_block(struct compact_control *cc,
|
| 576 | unsigned long *start_pfn,
|
| 577 | unsigned long end_pfn,
|
| 578 | struct list_head *freelist,
|
| 579 | unsigned int stride,
|
| 580 | bool strict)
|
| 581 | {
|
| 582 | int nr_scanned = 0, total_isolated = 0;
|
| 583 | struct page *page;
|
| 584 | unsigned long flags = 0;
|
| 585 | bool locked = false;
|
| 586 | unsigned long blockpfn = *start_pfn;
|
| 587 | unsigned int order;
|
| 588 |
|
| 589 |
|
| 590 | if (strict)
|
| 591 | stride = 1;
|
| 592 |
|
| 593 | page = pfn_to_page(blockpfn);
|
| 594 |
|
| 595 |
|
| 596 | for (; blockpfn < end_pfn; blockpfn += stride, page += stride) {
|
| 597 | int isolated;
|
| 598 |
|
| 599 | |
| 600 | * Periodically drop the lock (if held) regardless of its
|
| 601 | * contention, to give chance to IRQs. Abort if fatal signal
|
| 602 | * pending.
|
| 603 |
|
| 604 | if (!(blockpfn % COMPACT_CLUSTER_MAX)
|
| 605 | && compact_unlock_should_abort(&cc->zone->lock, flags,
|
| 606 | &locked, cc))
|
| 607 | break;
|
| 608 |
|
| 609 | nr_scanned++;
|
| 610 |
|
| 611 | |
| 612 | * For compound pages such as THP and hugetlbfs, we can save
|
| 613 | * potentially a lot of iterations if we skip them at once.
|
| 614 | * The check is racy, but we can consider only valid values
|
| 615 | * and the only danger is skipping too much.
|
| 616 |
|
| 617 | if (PageCompound(page)) {
|
| 618 | const unsigned int order = compound_order(page);
|
| 619 |
|
| 620 | if ((order <= MAX_PAGE_ORDER) &&
|
| 621 | (blockpfn + (1UL << order) <= end_pfn)) {
|
| 622 | blockpfn += (1UL << order) - 1;
|
| 623 | page += (1UL << order) - 1;
|
| 624 | nr_scanned += (1UL << order) - 1;
|
| 625 | }
|
| 626 |
|
| 627 | goto isolate_fail;
|
| 628 | }
|
| 629 |
|
| 630 | if (!PageBuddy(page))
|
| 631 | goto isolate_fail;
|
| 632 |
|
| 633 |
|
| 634 | if (!locked) {
|
| 635 | locked = compact_lock_irqsave(&cc->zone->lock,
|
| 636 | &flags, cc);
|
| 637 |
|
| 638 |
|
| 639 | if (!PageBuddy(page))
|
| 640 | goto isolate_fail;
|
| 641 | }
|
| 642 |
|
| 643 |
|
| 644 | order = buddy_order(page);
|
| 645 | isolated = __isolate_free_page(page, order);
|
| 646 | if (!isolated)
|
| 647 | break;
|
| 648 |
|
| 649 | nr_scanned += isolated - 1;
|
| 650 | total_isolated += isolated;
|
| 651 | cc->nr_freepages += isolated;
|
| 652 | list_add_tail(&page->lru, &freelist[order]);
|
| 653 |
|
| 654 | if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
|
| 655 | blockpfn += isolated;
|
| 656 | break;
|
| 657 | }
|
| 658 |
|
| 659 | blockpfn += isolated - 1;
|
| 660 | page += isolated - 1;
|
| 661 | continue;
|
| 662 |
|
| 663 | isolate_fail:
|
| 664 | if (strict)
|
| 665 | break;
|
| 666 |
|
| 667 | }
|
| 668 |
|
| 669 | if (locked)
|
| 670 | spin_unlock_irqrestore(&cc->zone->lock, flags);
|
| 671 |
|
| 672 | |
| 673 | * Be careful to not go outside of the pageblock.
|
| 674 |
|
| 675 | if (unlikely(blockpfn > end_pfn))
|
| 676 | blockpfn = end_pfn;
|
| 677 |
|
| 678 | trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
|
| 679 | nr_scanned, total_isolated);
|
| 680 |
|
| 681 |
|
| 682 | *start_pfn = blockpfn;
|
| 683 |
|
| 684 | |
| 685 | * If strict isolation is requested by CMA then check that all the
|
| 686 | * pages requested were isolated. If there were any failures, 0 is
|
| 687 | * returned and CMA will fail.
|
| 688 |
|
| 689 | if (strict && blockpfn < end_pfn)
|
| 690 | total_isolated = 0;
|
| 691 |
|
| 692 | cc->total_free_scanned += nr_scanned;
|
| 693 | if (total_isolated)
|
| 694 | count_compact_events(COMPACTISOLATED, total_isolated);
|
| 695 | return total_isolated;
|
| 696 | }
|
| 697 |
|
| 698 | |
| 699 | * isolate_freepages_range() - isolate free pages.
|
| 700 | * @cc: Compaction control structure.
|
| 701 | * @start_pfn: The first PFN to start isolating.
|
| 702 | * @end_pfn: The one-past-last PFN.
|
| 703 | *
|
| 704 | * Non-free pages, invalid PFNs, or zone boundaries within the
|
| 705 | * [start_pfn, end_pfn) range are considered errors, cause function to
|
| 706 | * undo its actions and return zero. cc->freepages[] are empty.
|
| 707 | *
|
| 708 | * Otherwise, function returns one-past-the-last PFN of isolated page
|
| 709 | * (which may be greater then end_pfn if end fell in a middle of
|
| 710 | * a free page). cc->freepages[] contain free pages isolated.
|
| 711 |
|
| 712 | unsigned long
|
| 713 | isolate_freepages_range(struct compact_control *cc,
|
| 714 | unsigned long start_pfn, unsigned long end_pfn)
|
| 715 | {
|
| 716 | unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
|
| 717 | int order;
|
| 718 |
|
| 719 | for (order = 0; order < NR_PAGE_ORDERS; order++)
|
| 720 | INIT_LIST_HEAD(&cc->freepages[order]);
|
| 721 |
|
| 722 | pfn = start_pfn;
|
| 723 | block_start_pfn = pageblock_start_pfn(pfn);
|
| 724 | if (block_start_pfn < cc->zone->zone_start_pfn)
|
| 725 | block_start_pfn = cc->zone->zone_start_pfn;
|
| 726 | block_end_pfn = pageblock_end_pfn(pfn);
|
| 727 |
|
| 728 | for (; pfn < end_pfn; pfn += isolated,
|
| 729 | block_start_pfn = block_end_pfn,
|
| 730 | block_end_pfn += pageblock_nr_pages) {
|
| 731 |
|
| 732 | unsigned long isolate_start_pfn = pfn;
|
| 733 |
|
| 734 | |
| 735 | * pfn could pass the block_end_pfn if isolated freepage
|
| 736 | * is more than pageblock order. In this case, we adjust
|
| 737 | * scanning range to right one.
|
| 738 |
|
| 739 | if (pfn >= block_end_pfn) {
|
| 740 | block_start_pfn = pageblock_start_pfn(pfn);
|
| 741 | block_end_pfn = pageblock_end_pfn(pfn);
|
| 742 | }
|
| 743 |
|
| 744 | block_end_pfn = min(block_end_pfn, end_pfn);
|
| 745 |
|
| 746 | if (!pageblock_pfn_to_page(block_start_pfn,
|
| 747 | block_end_pfn, cc->zone))
|
| 748 | break;
|
| 749 |
|
| 750 | isolated = isolate_freepages_block(cc, &isolate_start_pfn,
|
| 751 | block_end_pfn, cc->freepages, 0, true);
|
| 752 |
|
| 753 | |
| 754 | * In strict mode, isolate_freepages_block() returns 0 if
|
| 755 | * there are any holes in the block (ie. invalid PFNs or
|
| 756 | * non-free pages).
|
| 757 |
|
| 758 | if (!isolated)
|
| 759 | break;
|
| 760 |
|
| 761 | |
| 762 | * If we managed to isolate pages, it is always (1 << n) *
|
| 763 | * pageblock_nr_pages for some non-negative n. (Max order
|
| 764 | * page may span two pageblocks).
|
| 765 |
|
| 766 | }
|
| 767 |
|
| 768 | if (pfn < end_pfn) {
|
| 769 |
|
| 770 | release_free_list(cc->freepages);
|
| 771 | return 0;
|
| 772 | }
|
| 773 |
|
| 774 |
|
| 775 | return pfn;
|
| 776 | }
|
| 777 |
|
| 778 |
|
| 779 | static bool too_many_isolated(struct compact_control *cc)
|
| 780 | {
|
| 781 | pg_data_t *pgdat = cc->zone->zone_pgdat;
|
| 782 | bool too_many;
|
| 783 |
|
| 784 | unsigned long active, inactive, isolated;
|
| 785 |
|
| 786 | inactive = node_page_state(pgdat, NR_INACTIVE_FILE) +
|
| 787 | node_page_state(pgdat, NR_INACTIVE_ANON);
|
| 788 | active = node_page_state(pgdat, NR_ACTIVE_FILE) +
|
| 789 | node_page_state(pgdat, NR_ACTIVE_ANON);
|
| 790 | isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
|
| 791 | node_page_state(pgdat, NR_ISOLATED_ANON);
|
| 792 |
|
| 793 | |
| 794 | * Allow GFP_NOFS to isolate past the limit set for regular
|
| 795 | * compaction runs. This prevents an ABBA deadlock when other
|
| 796 | * compactors have already isolated to the limit, but are
|
| 797 | * blocked on filesystem locks held by the GFP_NOFS thread.
|
| 798 |
|
| 799 | if (cc->gfp_mask & __GFP_FS) {
|
| 800 | inactive >>= 3;
|
| 801 | active >>= 3;
|
| 802 | }
|
| 803 |
|
| 804 | too_many = isolated > (inactive + active) / 2;
|
| 805 | if (!too_many)
|
| 806 | wake_throttle_isolated(pgdat);
|
| 807 |
|
| 808 | return too_many;
|
| 809 | }
|
| 810 |
|
| 811 | |
| 812 | * skip_isolation_on_order() - determine when to skip folio isolation based on
|
| 813 | * folio order and compaction target order
|
| 814 | * @order: to-be-isolated folio order
|
| 815 | * @target_order: compaction target order
|
| 816 | *
|
| 817 | * This avoids unnecessary folio isolations during compaction.
|
| 818 |
|
| 819 | static bool skip_isolation_on_order(int order, int target_order)
|
| 820 | {
|
| 821 | |
| 822 | * Unless we are performing global compaction (i.e.,
|
| 823 | * is_via_compact_memory), skip any folios that are larger than the
|
| 824 | * target order: we wouldn't be here if we'd have a free folio with
|
| 825 | * the desired target_order, so migrating this folio would likely fail
|
| 826 | * later.
|
| 827 |
|
| 828 | if (!is_via_compact_memory(target_order) && order >= target_order)
|
| 829 | return true;
|
| 830 | |
| 831 | * We limit memory compaction to pageblocks and won't try
|
| 832 | * creating free blocks of memory that are larger than that.
|
| 833 |
|
| 834 | return order >= pageblock_order;
|
| 835 | }
|
| 836 |
|
| 837 | |
| 838 | * isolate_migratepages_block() - isolate all migrate-able pages within
|
| 839 | * a single pageblock
|
| 840 | * @cc: Compaction control structure.
|
| 841 | * @low_pfn: The first PFN to isolate
|
| 842 | * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock
|
| 843 | * @mode: Isolation mode to be used.
|
| 844 | *
|
| 845 | * Isolate all pages that can be migrated from the range specified by
|
| 846 | * [low_pfn, end_pfn). The range is expected to be within same pageblock.
|
| 847 | * Returns errno, like -EAGAIN or -EINTR in case e.g signal pending or congestion,
|
| 848 | * -ENOMEM in case we could not allocate a page, or 0.
|
| 849 | * cc->migrate_pfn will contain the next pfn to scan.
|
| 850 | *
|
| 851 | * The pages are isolated on cc->migratepages list (not required to be empty),
|
| 852 | * and cc->nr_migratepages is updated accordingly.
|
| 853 |
|
| 854 | static int
|
| 855 | isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
|
| 856 | unsigned long end_pfn, isolate_mode_t mode)
|
| 857 | {
|
| 858 | pg_data_t *pgdat = cc->zone->zone_pgdat;
|
| 859 | unsigned long nr_scanned = 0, nr_isolated = 0;
|
| 860 | struct lruvec *lruvec = NULL;
|
| 861 | unsigned long flags = 0;
|
| 862 | struct lruvec *locked = NULL;
|
| 863 | struct folio *folio = NULL;
|
| 864 | struct page *page = NULL, *valid_page = NULL;
|
| 865 | struct address_space *mapping;
|
| 866 | unsigned long start_pfn = low_pfn;
|
| 867 | bool skip_on_failure = false;
|
| 868 | unsigned long next_skip_pfn = 0;
|
| 869 | bool skip_updated = false;
|
| 870 | int ret = 0;
|
| 871 |
|
| 872 | cc->migrate_pfn = low_pfn;
|
| 873 |
|
| 874 | |
| 875 | * Ensure that there are not too many pages isolated from the LRU
|
| 876 | * list by either parallel reclaimers or compaction. If there are,
|
| 877 | * delay for some time until fewer pages are isolated
|
| 878 |
|
| 879 | while (unlikely(too_many_isolated(cc))) {
|
| 880 |
|
| 881 | if (cc->nr_migratepages)
|
| 882 | return -EAGAIN;
|
| 883 |
|
| 884 |
|
| 885 | if (cc->mode == MIGRATE_ASYNC)
|
| 886 | return -EAGAIN;
|
| 887 |
|
| 888 | reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
|
| 889 |
|
| 890 | if (fatal_signal_pending(current))
|
| 891 | return -EINTR;
|
| 892 | }
|
| 893 |
|
| 894 | cond_resched();
|
| 895 |
|
| 896 | if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
|
| 897 | skip_on_failure = true;
|
| 898 | next_skip_pfn = block_end_pfn(low_pfn, cc->order);
|
| 899 | }
|
| 900 |
|
| 901 |
|
| 902 | for (; low_pfn < end_pfn; low_pfn++) {
|
| 903 | bool is_dirty, is_unevictable;
|
| 904 |
|
| 905 | if (skip_on_failure && low_pfn >= next_skip_pfn) {
|
| 906 | |
| 907 | * We have isolated all migration candidates in the
|
| 908 | * previous order-aligned block, and did not skip it due
|
| 909 | * to failure. We should migrate the pages now and
|
| 910 | * hopefully succeed compaction.
|
| 911 |
|
| 912 | if (nr_isolated)
|
| 913 | break;
|
| 914 |
|
| 915 | |
| 916 | * We failed to isolate in the previous order-aligned
|
| 917 | * block. Set the new boundary to the end of the
|
| 918 | * current block. Note we can't simply increase
|
| 919 | * next_skip_pfn by 1 << order, as low_pfn might have
|
| 920 | * been incremented by a higher number due to skipping
|
| 921 | * a compound or a high-order buddy page in the
|
| 922 | * previous loop iteration.
|
| 923 |
|
| 924 | next_skip_pfn = block_end_pfn(low_pfn, cc->order);
|
| 925 | }
|
| 926 |
|
| 927 | |
| 928 | * Periodically drop the lock (if held) regardless of its
|
| 929 | * contention, to give chance to IRQs. Abort completely if
|
| 930 | * a fatal signal is pending.
|
| 931 |
|
| 932 | if (!(low_pfn % COMPACT_CLUSTER_MAX)) {
|
| 933 | if (locked) {
|
| 934 | lruvec_unlock_irqrestore(locked, flags);
|
| 935 | locked = NULL;
|
| 936 | }
|
| 937 |
|
| 938 | if (fatal_signal_pending(current)) {
|
| 939 | cc->contended = true;
|
| 940 | ret = -EINTR;
|
| 941 |
|
| 942 | goto fatal_pending;
|
| 943 | }
|
| 944 |
|
| 945 | cond_resched();
|
| 946 | }
|
| 947 |
|
| 948 | nr_scanned++;
|
| 949 |
|
| 950 | page = pfn_to_page(low_pfn);
|
| 951 |
|
| 952 | |
| 953 | * Check if the pageblock has already been marked skipped.
|
| 954 | * Only the first PFN is checked as the caller isolates
|
| 955 | * COMPACT_CLUSTER_MAX at a time so the second call must
|
| 956 | * not falsely conclude that the block should be skipped.
|
| 957 |
|
| 958 | if (!valid_page && (pageblock_aligned(low_pfn) ||
|
| 959 | low_pfn == cc->zone->zone_start_pfn)) {
|
| 960 | if (!isolation_suitable(cc, page)) {
|
| 961 | low_pfn = end_pfn;
|
| 962 | folio = NULL;
|
| 963 | goto isolate_abort;
|
| 964 | }
|
| 965 | valid_page = page;
|
| 966 | }
|
| 967 |
|
| 968 | if (PageHuge(page)) {
|
| 969 | const unsigned int order = compound_order(page);
|
| 970 | |
| 971 | * skip hugetlbfs if we are not compacting for pages
|
| 972 | * bigger than its order. THPs and other compound pages
|
| 973 | * are handled below.
|
| 974 |
|
| 975 | if (!cc->alloc_contig) {
|
| 976 |
|
| 977 | if (order <= MAX_PAGE_ORDER) {
|
| 978 | low_pfn += (1UL << order) - 1;
|
| 979 | nr_scanned += (1UL << order) - 1;
|
| 980 | }
|
| 981 | goto isolate_fail;
|
| 982 | }
|
| 983 |
|
| 984 | if (locked) {
|
| 985 | lruvec_unlock_irqrestore(locked, flags);
|
| 986 | locked = NULL;
|
| 987 | }
|
| 988 |
|
| 989 | folio = page_folio(page);
|
| 990 | ret = isolate_or_dissolve_huge_folio(folio, &cc->migratepages);
|
| 991 |
|
| 992 | |
| 993 | * Fail isolation in case isolate_or_dissolve_huge_folio()
|
| 994 | * reports an error. In case of -ENOMEM, abort right away.
|
| 995 |
|
| 996 | if (ret < 0) {
|
| 997 |
|
| 998 | if (ret == -EBUSY)
|
| 999 | ret = 0;
|
| 1000 | low_pfn += (1UL << order) - 1;
|
| 1001 | nr_scanned += (1UL << order) - 1;
|
| 1002 | goto isolate_fail;
|
| 1003 | }
|
| 1004 |
|
| 1005 | if (folio_test_hugetlb(folio)) {
|
| 1006 | |
| 1007 | * Hugepage was successfully isolated and placed
|
| 1008 | * on the cc->migratepages list.
|
| 1009 |
|
| 1010 | low_pfn += folio_nr_pages(folio) - folio_page_idx(folio, page) - 1;
|
| 1011 | goto isolate_success_no_list;
|
| 1012 | }
|
| 1013 |
|
| 1014 | |
| 1015 | * Ok, the hugepage was dissolved. Now these pages are
|
| 1016 | * Buddy and cannot be re-allocated because they are
|
| 1017 | * isolated. Fall-through as the check below handles
|
| 1018 | * Buddy pages.
|
| 1019 |
|
| 1020 | }
|
| 1021 |
|
| 1022 | |
| 1023 | * Skip if free. We read page order here without zone lock
|
| 1024 | * which is generally unsafe, but the race window is small and
|
| 1025 | * the worst thing that can happen is that we skip some
|
| 1026 | * potential isolation targets.
|
| 1027 |
|
| 1028 | if (PageBuddy(page)) {
|
| 1029 | unsigned long freepage_order = buddy_order_unsafe(page);
|
| 1030 |
|
| 1031 | |
| 1032 | * Without lock, we cannot be sure that what we got is
|
| 1033 | * a valid page order. Consider only values in the
|
| 1034 | * valid order range to prevent low_pfn overflow.
|
| 1035 |
|
| 1036 | if (freepage_order > 0 && freepage_order <= MAX_PAGE_ORDER) {
|
| 1037 | low_pfn += (1UL << freepage_order) - 1;
|
| 1038 | nr_scanned += (1UL << freepage_order) - 1;
|
| 1039 | }
|
| 1040 | continue;
|
| 1041 | }
|
| 1042 |
|
| 1043 | |
| 1044 | * Regardless of being on LRU, compound pages such as THP
|
| 1045 | * (hugetlbfs is handled above) are not to be compacted unless
|
| 1046 | * we are attempting an allocation larger than the compound
|
| 1047 | * page size. We can potentially save a lot of iterations if we
|
| 1048 | * skip them at once. The check is racy, but we can consider
|
| 1049 | * only valid values and the only danger is skipping too much.
|
| 1050 |
|
| 1051 | if (PageCompound(page) && !cc->alloc_contig) {
|
| 1052 | const unsigned int order = compound_order(page);
|
| 1053 |
|
| 1054 |
|
| 1055 | if (skip_isolation_on_order(order, cc->order)) {
|
| 1056 | if (order <= MAX_PAGE_ORDER) {
|
| 1057 | low_pfn += (1UL << order) - 1;
|
| 1058 | nr_scanned += (1UL << order) - 1;
|
| 1059 | }
|
| 1060 | goto isolate_fail;
|
| 1061 | }
|
| 1062 | }
|
| 1063 |
|
| 1064 | |
| 1065 | * Check may be lockless but that's ok as we recheck later.
|
| 1066 | * It's possible to migrate LRU and non-lru movable pages.
|
| 1067 | * Skip any other type of page
|
| 1068 |
|
| 1069 | if (!PageLRU(page)) {
|
| 1070 |
|
| 1071 | if (unlikely(page_has_movable_ops(page)) &&
|
| 1072 | !PageMovableOpsIsolated(page)) {
|
| 1073 | if (locked) {
|
| 1074 | lruvec_unlock_irqrestore(locked, flags);
|
| 1075 | locked = NULL;
|
| 1076 | }
|
| 1077 |
|
| 1078 | if (isolate_movable_ops_page(page, mode)) {
|
| 1079 | folio = page_folio(page);
|
| 1080 | goto isolate_success;
|
| 1081 | }
|
| 1082 | }
|
| 1083 |
|
| 1084 | goto isolate_fail;
|
| 1085 | }
|
| 1086 |
|
| 1087 | |
| 1088 | * Be careful not to clear PageLRU until after we're
|
| 1089 | * sure the page is not being freed elsewhere -- the
|
| 1090 | * page release code relies on it.
|
| 1091 |
|
| 1092 | folio = folio_get_nontail_page(page);
|
| 1093 | if (unlikely(!folio))
|
| 1094 | goto isolate_fail;
|
| 1095 |
|
| 1096 | |
| 1097 | * Migration will fail if an anonymous page is pinned in memory,
|
| 1098 | * so avoid taking lru_lock and isolating it unnecessarily in an
|
| 1099 | * admittedly racy check.
|
| 1100 |
|
| 1101 | mapping = folio_mapping(folio);
|
| 1102 | if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))
|
| 1103 | goto isolate_fail_put;
|
| 1104 |
|
| 1105 | |
| 1106 | * Only allow to migrate anonymous pages in GFP_NOFS context
|
| 1107 | * because those do not depend on fs locks.
|
| 1108 |
|
| 1109 | if (!(cc->gfp_mask & __GFP_FS) && mapping)
|
| 1110 | goto isolate_fail_put;
|
| 1111 |
|
| 1112 |
|
| 1113 | if (!folio_test_lru(folio))
|
| 1114 | goto isolate_fail_put;
|
| 1115 |
|
| 1116 | is_unevictable = folio_test_unevictable(folio);
|
| 1117 |
|
| 1118 |
|
| 1119 | if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
|
| 1120 | goto isolate_fail_put;
|
| 1121 |
|
| 1122 | |
| 1123 | * To minimise LRU disruption, the caller can indicate with
|
| 1124 | * ISOLATE_ASYNC_MIGRATE that it only wants to isolate pages
|
| 1125 | * it will be able to migrate without blocking - clean pages
|
| 1126 | * for the most part. Writeback would require blocking.
|
| 1127 |
|
| 1128 | if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
|
| 1129 | goto isolate_fail_put;
|
| 1130 |
|
| 1131 | is_dirty = folio_test_dirty(folio);
|
| 1132 |
|
| 1133 | if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) ||
|
| 1134 | (mapping && is_unevictable)) {
|
| 1135 | bool migrate_dirty = true;
|
| 1136 | bool is_inaccessible;
|
| 1137 |
|
| 1138 | |
| 1139 | * Only folios without mappings or that have
|
| 1140 | * a ->migrate_folio callback are possible to migrate
|
| 1141 | * without blocking.
|
| 1142 | *
|
| 1143 | * Folios from inaccessible mappings are not migratable.
|
| 1144 | *
|
| 1145 | * However, we can be racing with truncation, which can
|
| 1146 | * free the mapping that we need to check. Truncation
|
| 1147 | * holds the folio lock until after the folio is removed
|
| 1148 | * from the page so holding it ourselves is sufficient.
|
| 1149 | *
|
| 1150 | * To avoid locking the folio just to check inaccessible,
|
| 1151 | * assume every inaccessible folio is also unevictable,
|
| 1152 | * which is a cheaper test. If our assumption goes
|
| 1153 | * wrong, it's not a correctness bug, just potentially
|
| 1154 | * wasted cycles.
|
| 1155 |
|
| 1156 | if (!folio_trylock(folio))
|
| 1157 | goto isolate_fail_put;
|
| 1158 |
|
| 1159 | mapping = folio_mapping(folio);
|
| 1160 | if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) {
|
| 1161 | migrate_dirty = !mapping ||
|
| 1162 | mapping->a_ops->migrate_folio;
|
| 1163 | }
|
| 1164 | is_inaccessible = mapping && mapping_inaccessible(mapping);
|
| 1165 | folio_unlock(folio);
|
| 1166 | if (!migrate_dirty || is_inaccessible)
|
| 1167 | goto isolate_fail_put;
|
| 1168 | }
|
| 1169 |
|
| 1170 |
|
| 1171 | if (!folio_test_clear_lru(folio))
|
| 1172 | goto isolate_fail_put;
|
| 1173 |
|
| 1174 | if (locked)
|
| 1175 | lruvec = folio_lruvec(folio);
|
| 1176 |
|
| 1177 |
|
| 1178 | if (lruvec != locked || !locked) {
|
| 1179 | if (locked)
|
| 1180 | lruvec_unlock_irqrestore(locked, flags);
|
| 1181 |
|
| 1182 | lruvec = compact_folio_lruvec_lock_irqsave(folio, &flags, cc);
|
| 1183 | locked = lruvec;
|
| 1184 |
|
| 1185 | |
| 1186 | * Try get exclusive access under lock. If marked for
|
| 1187 | * skip, the scan is aborted unless the current context
|
| 1188 | * is a rescan to reach the end of the pageblock.
|
| 1189 |
|
| 1190 | if (!skip_updated && valid_page) {
|
| 1191 | skip_updated = true;
|
| 1192 | if (test_and_set_skip(cc, valid_page) &&
|
| 1193 | !cc->finish_pageblock) {
|
| 1194 | low_pfn = end_pfn;
|
| 1195 | goto isolate_abort;
|
| 1196 | }
|
| 1197 | }
|
| 1198 |
|
| 1199 | |
| 1200 | * Check LRU folio order under the lock
|
| 1201 |
|
| 1202 | if (unlikely(skip_isolation_on_order(folio_order(folio),
|
| 1203 | cc->order) &&
|
| 1204 | !cc->alloc_contig)) {
|
| 1205 | low_pfn += folio_nr_pages(folio) - 1;
|
| 1206 | nr_scanned += folio_nr_pages(folio) - 1;
|
| 1207 | folio_set_lru(folio);
|
| 1208 | goto isolate_fail_put;
|
| 1209 | }
|
| 1210 | }
|
| 1211 |
|
| 1212 |
|
| 1213 | if (folio_test_large(folio))
|
| 1214 | low_pfn += folio_nr_pages(folio) - 1;
|
| 1215 |
|
| 1216 |
|
| 1217 | lruvec_del_folio(lruvec, folio);
|
| 1218 | node_stat_mod_folio(folio,
|
| 1219 | NR_ISOLATED_ANON + folio_is_file_lru(folio),
|
| 1220 | folio_nr_pages(folio));
|
| 1221 |
|
| 1222 | isolate_success:
|
| 1223 | list_add(&folio->lru, &cc->migratepages);
|
| 1224 | isolate_success_no_list:
|
| 1225 | cc->nr_migratepages += folio_nr_pages(folio);
|
| 1226 | nr_isolated += folio_nr_pages(folio);
|
| 1227 | nr_scanned += folio_nr_pages(folio) - 1;
|
| 1228 |
|
| 1229 | |
| 1230 | * Avoid isolating too much unless this block is being
|
| 1231 | * fully scanned (e.g. dirty/writeback pages, parallel allocation)
|
| 1232 | * or a lock is contended. For contention, isolate quickly to
|
| 1233 | * potentially remove one source of contention.
|
| 1234 |
|
| 1235 | if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX &&
|
| 1236 | !cc->finish_pageblock && !cc->contended) {
|
| 1237 | ++low_pfn;
|
| 1238 | break;
|
| 1239 | }
|
| 1240 |
|
| 1241 | continue;
|
| 1242 |
|
| 1243 | isolate_fail_put:
|
| 1244 |
|
| 1245 | if (locked) {
|
| 1246 | lruvec_unlock_irqrestore(locked, flags);
|
| 1247 | locked = NULL;
|
| 1248 | }
|
| 1249 | folio_put(folio);
|
| 1250 |
|
| 1251 | isolate_fail:
|
| 1252 | if (!skip_on_failure && ret != -ENOMEM)
|
| 1253 | continue;
|
| 1254 |
|
| 1255 | |
| 1256 | * We have isolated some pages, but then failed. Release them
|
| 1257 | * instead of migrating, as we cannot form the cc->order buddy
|
| 1258 | * page anyway.
|
| 1259 |
|
| 1260 | if (nr_isolated) {
|
| 1261 | if (locked) {
|
| 1262 | lruvec_unlock_irqrestore(locked, flags);
|
| 1263 | locked = NULL;
|
| 1264 | }
|
| 1265 | putback_movable_pages(&cc->migratepages);
|
| 1266 | cc->nr_migratepages = 0;
|
| 1267 | nr_isolated = 0;
|
| 1268 | }
|
| 1269 |
|
| 1270 | if (low_pfn < next_skip_pfn) {
|
| 1271 | low_pfn = next_skip_pfn - 1;
|
| 1272 | |
| 1273 | * The check near the loop beginning would have updated
|
| 1274 | * next_skip_pfn too, but this is a bit simpler.
|
| 1275 |
|
| 1276 | next_skip_pfn += 1UL << cc->order;
|
| 1277 | }
|
| 1278 |
|
| 1279 | if (ret == -ENOMEM)
|
| 1280 | break;
|
| 1281 | }
|
| 1282 |
|
| 1283 | |
| 1284 | * The PageBuddy() check could have potentially brought us outside
|
| 1285 | * the range to be scanned.
|
| 1286 |
|
| 1287 | if (unlikely(low_pfn > end_pfn))
|
| 1288 | low_pfn = end_pfn;
|
| 1289 |
|
| 1290 | folio = NULL;
|
| 1291 |
|
| 1292 | isolate_abort:
|
| 1293 | if (locked)
|
| 1294 | lruvec_unlock_irqrestore(locked, flags);
|
| 1295 | if (folio) {
|
| 1296 | folio_set_lru(folio);
|
| 1297 | folio_put(folio);
|
| 1298 | }
|
| 1299 |
|
| 1300 | |
| 1301 | * Update the cached scanner pfn once the pageblock has been scanned.
|
| 1302 | * Pages will either be migrated in which case there is no point
|
| 1303 | * scanning in the near future or migration failed in which case the
|
| 1304 | * failure reason may persist. The block is marked for skipping if
|
| 1305 | * there were no pages isolated in the block or if the block is
|
| 1306 | * rescanned twice in a row.
|
| 1307 |
|
| 1308 | if (low_pfn == end_pfn && (!nr_isolated || cc->finish_pageblock)) {
|
| 1309 | if (!cc->no_set_skip_hint && valid_page && !skip_updated)
|
| 1310 | set_pageblock_skip(valid_page);
|
| 1311 | update_cached_migrate(cc, low_pfn);
|
| 1312 | }
|
| 1313 |
|
| 1314 | trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
|
| 1315 | nr_scanned, nr_isolated);
|
| 1316 |
|
| 1317 | fatal_pending:
|
| 1318 | cc->total_migrate_scanned += nr_scanned;
|
| 1319 | if (nr_isolated)
|
| 1320 | count_compact_events(COMPACTISOLATED, nr_isolated);
|
| 1321 |
|
| 1322 | cc->migrate_pfn = low_pfn;
|
| 1323 |
|
| 1324 | return ret;
|
| 1325 | }
|
| 1326 |
|
| 1327 | |
| 1328 | * isolate_migratepages_range() - isolate migrate-able pages in a PFN range
|
| 1329 | * @cc: Compaction control structure.
|
| 1330 | * @start_pfn: The first PFN to start isolating.
|
| 1331 | * @end_pfn: The one-past-last PFN.
|
| 1332 | *
|
| 1333 | * Returns -EAGAIN when contented, -EINTR in case of a signal pending, -ENOMEM
|
| 1334 | * in case we could not allocate a page, or 0.
|
| 1335 |
|
| 1336 | int
|
| 1337 | isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
|
| 1338 | unsigned long end_pfn)
|
| 1339 | {
|
| 1340 | unsigned long pfn, block_start_pfn, block_end_pfn;
|
| 1341 | int ret = 0;
|
| 1342 |
|
| 1343 |
|
| 1344 | pfn = start_pfn;
|
| 1345 | block_start_pfn = pageblock_start_pfn(pfn);
|
| 1346 | if (block_start_pfn < cc->zone->zone_start_pfn)
|
| 1347 | block_start_pfn = cc->zone->zone_start_pfn;
|
| 1348 | block_end_pfn = pageblock_end_pfn(pfn);
|
| 1349 |
|
| 1350 | for (; pfn < end_pfn; pfn = block_end_pfn,
|
| 1351 | block_start_pfn = block_end_pfn,
|
| 1352 | block_end_pfn += pageblock_nr_pages) {
|
| 1353 |
|
| 1354 | block_end_pfn = min(block_end_pfn, end_pfn);
|
| 1355 |
|
| 1356 | if (!pageblock_pfn_to_page(block_start_pfn,
|
| 1357 | block_end_pfn, cc->zone))
|
| 1358 | continue;
|
| 1359 |
|
| 1360 | ret = isolate_migratepages_block(cc, pfn, block_end_pfn,
|
| 1361 | ISOLATE_UNEVICTABLE);
|
| 1362 |
|
| 1363 | if (ret)
|
| 1364 | break;
|
| 1365 |
|
| 1366 | if (cc->nr_migratepages >= COMPACT_CLUSTER_MAX)
|
| 1367 | break;
|
| 1368 | }
|
| 1369 |
|
| 1370 | return ret;
|
| 1371 | }
|
| 1372 |
|
| 1373 | #endif
|
| 1374 | #ifdef CONFIG_COMPACTION
|
| 1375 |
|
| 1376 | static bool suitable_migration_source(struct compact_control *cc,
|
| 1377 | struct page *page)
|
| 1378 | {
|
| 1379 | int block_mt;
|
| 1380 |
|
| 1381 | if (pageblock_skip_persistent(page))
|
| 1382 | return false;
|
| 1383 |
|
| 1384 | |
| 1385 | * Background compaction produces blocks for the zone at
|
| 1386 | * large, with no particular allocation context. Allow all
|
| 1387 | * block types, including CMA.
|
| 1388 |
|
| 1389 | if (!cc->direct_compaction)
|
| 1390 | return true;
|
| 1391 |
|
| 1392 | block_mt = get_pageblock_migratetype(page);
|
| 1393 |
|
| 1394 | |
| 1395 | * CMA pages can only be taken by ALLOC_CMA requests. For anybody
|
| 1396 | * else, vacating a CMA block consumes free pages the caller
|
| 1397 | * could have used, and produces free pages it cannot.
|
| 1398 |
|
| 1399 | if (is_migrate_cma(block_mt) && !(cc->alloc_flags & ALLOC_CMA))
|
| 1400 | return false;
|
| 1401 |
|
| 1402 | |
| 1403 | * Per default, scans are restricted to blocks compatible with
|
| 1404 | * the request, to prevent cross-contamination. Once
|
| 1405 | * compaction priority escalates to synchronous scans, though,
|
| 1406 | * scan all blocks to try to make forward progress. For
|
| 1407 | * movable request, this likely helps little: there shouldn't
|
| 1408 | * be many migratable pages inside non-movable blocks besides
|
| 1409 | * allocator fallbacks. For non-movable requests, this helps a
|
| 1410 | * lot, as they can finally scan movable blocks.
|
| 1411 |
|
| 1412 | if (cc->mode != MIGRATE_ASYNC)
|
| 1413 | return true;
|
| 1414 |
|
| 1415 | |
| 1416 | * Prevent <pageblock_order unmovable/reclaimable requests from
|
| 1417 | * polluting movable blocks through fallbacks. Whole-block production
|
| 1418 | * (directly requested, or defrag_mode) is exempt as the allocator
|
| 1419 | * claims and converts these.
|
| 1420 |
|
| 1421 | if (cc->migratetype == MIGRATE_MOVABLE || cc->order >= pageblock_order)
|
| 1422 | return is_migrate_movable(block_mt);
|
| 1423 | else
|
| 1424 | return block_mt == cc->migratetype;
|
| 1425 | }
|
| 1426 |
|
| 1427 |
|
| 1428 | static bool suitable_migration_target(struct compact_control *cc,
|
| 1429 | struct page *page)
|
| 1430 | {
|
| 1431 |
|
| 1432 | if (PageBuddy(page)) {
|
| 1433 | int order = cc->order > 0 ? cc->order : pageblock_order;
|
| 1434 |
|
| 1435 | |
| 1436 | * We are checking page_order without zone->lock taken. But
|
| 1437 | * the only small danger is that we skip a potentially suitable
|
| 1438 | * pageblock, so it's not worth to check order for valid range.
|
| 1439 |
|
| 1440 | if (buddy_order_unsafe(page) >= order)
|
| 1441 | return false;
|
| 1442 | }
|
| 1443 |
|
| 1444 | if (cc->ignore_block_suitable)
|
| 1445 | return true;
|
| 1446 |
|
| 1447 |
|
| 1448 | if (is_migrate_movable(get_pageblock_migratetype(page)))
|
| 1449 | return true;
|
| 1450 |
|
| 1451 |
|
| 1452 | return false;
|
| 1453 | }
|
| 1454 |
|
| 1455 | static inline unsigned int
|
| 1456 | freelist_scan_limit(struct compact_control *cc)
|
| 1457 | {
|
| 1458 | unsigned short shift = BITS_PER_LONG - 1;
|
| 1459 |
|
| 1460 | return (COMPACT_CLUSTER_MAX >> min(shift, cc->fast_search_fail)) + 1;
|
| 1461 | }
|
| 1462 |
|
| 1463 | |
| 1464 | * Test whether the free scanner has reached the same or lower pageblock than
|
| 1465 | * the migration scanner, and compaction should thus terminate.
|
| 1466 |
|
| 1467 | static inline bool compact_scanners_met(struct compact_control *cc)
|
| 1468 | {
|
| 1469 | return (cc->free_pfn >> pageblock_order)
|
| 1470 | <= (cc->migrate_pfn >> pageblock_order);
|
| 1471 | }
|
| 1472 |
|
| 1473 | |
| 1474 | * Used when scanning for a suitable migration target which scans freelists
|
| 1475 | * in reverse. Reorders the list such as the unscanned pages are scanned
|
| 1476 | * first on the next iteration of the free scanner
|
| 1477 |
|
| 1478 | static void
|
| 1479 | move_freelist_head(struct list_head *freelist, struct page *freepage)
|
| 1480 | {
|
| 1481 | LIST_HEAD(sublist);
|
| 1482 |
|
| 1483 | if (!list_is_first(&freepage->buddy_list, freelist)) {
|
| 1484 | list_cut_before(&sublist, freelist, &freepage->buddy_list);
|
| 1485 | list_splice_tail(&sublist, freelist);
|
| 1486 | }
|
| 1487 | }
|
| 1488 |
|
| 1489 | |
| 1490 | * Similar to move_freelist_head except used by the migration scanner
|
| 1491 | * when scanning forward. It's possible for these list operations to
|
| 1492 | * move against each other if they search the free list exactly in
|
| 1493 | * lockstep.
|
| 1494 |
|
| 1495 | static void
|
| 1496 | move_freelist_tail(struct list_head *freelist, struct page *freepage)
|
| 1497 | {
|
| 1498 | LIST_HEAD(sublist);
|
| 1499 |
|
| 1500 | if (!list_is_last(&freepage->buddy_list, freelist)) {
|
| 1501 | list_cut_position(&sublist, freelist, &freepage->buddy_list);
|
| 1502 | list_splice_tail(&sublist, freelist);
|
| 1503 | }
|
| 1504 | }
|
| 1505 |
|
| 1506 | static void
|
| 1507 | fast_isolate_around(struct compact_control *cc, unsigned long pfn)
|
| 1508 | {
|
| 1509 | unsigned long start_pfn, end_pfn;
|
| 1510 | struct page *page;
|
| 1511 |
|
| 1512 |
|
| 1513 | if (cc->nr_freepages >= cc->nr_migratepages)
|
| 1514 | return;
|
| 1515 |
|
| 1516 |
|
| 1517 | if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
|
| 1518 | return;
|
| 1519 |
|
| 1520 |
|
| 1521 | start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);
|
| 1522 | end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));
|
| 1523 |
|
| 1524 | page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);
|
| 1525 | if (!page)
|
| 1526 | return;
|
| 1527 |
|
| 1528 | isolate_freepages_block(cc, &start_pfn, end_pfn, cc->freepages, 1, false);
|
| 1529 |
|
| 1530 |
|
| 1531 | if (start_pfn == end_pfn && !cc->no_set_skip_hint)
|
| 1532 | set_pageblock_skip(page);
|
| 1533 | }
|
| 1534 |
|
| 1535 |
|
| 1536 | static int next_search_order(struct compact_control *cc, int order)
|
| 1537 | {
|
| 1538 | order--;
|
| 1539 | if (order < 0)
|
| 1540 | order = cc->order - 1;
|
| 1541 |
|
| 1542 |
|
| 1543 | if (order == cc->search_order) {
|
| 1544 | cc->search_order--;
|
| 1545 | if (cc->search_order < 0)
|
| 1546 | cc->search_order = cc->order - 1;
|
| 1547 | return -1;
|
| 1548 | }
|
| 1549 |
|
| 1550 | return order;
|
| 1551 | }
|
| 1552 |
|
| 1553 | static void fast_isolate_freepages(struct compact_control *cc)
|
| 1554 | {
|
| 1555 | unsigned int limit = max(1U, freelist_scan_limit(cc) >> 1);
|
| 1556 | unsigned int nr_scanned = 0, total_isolated = 0;
|
| 1557 | unsigned long low_pfn, min_pfn, highest = 0;
|
| 1558 | unsigned long nr_isolated = 0;
|
| 1559 | unsigned long distance;
|
| 1560 | struct page *page = NULL;
|
| 1561 | bool scan_start = false;
|
| 1562 | int order;
|
| 1563 |
|
| 1564 |
|
| 1565 | if (cc->order <= 0)
|
| 1566 | return;
|
| 1567 |
|
| 1568 | |
| 1569 | * If starting the scan, use a deeper search and use the highest
|
| 1570 | * PFN found if a suitable one is not found.
|
| 1571 |
|
| 1572 | if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {
|
| 1573 | limit = pageblock_nr_pages >> 1;
|
| 1574 | scan_start = true;
|
| 1575 | }
|
| 1576 |
|
| 1577 | |
| 1578 | * Preferred point is in the top quarter of the scan space but take
|
| 1579 | * a pfn from the top half if the search is problematic.
|
| 1580 |
|
| 1581 | distance = (cc->free_pfn - cc->migrate_pfn);
|
| 1582 | low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
|
| 1583 | min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));
|
| 1584 |
|
| 1585 | if (WARN_ON_ONCE(min_pfn > low_pfn))
|
| 1586 | low_pfn = min_pfn;
|
| 1587 |
|
| 1588 | |
| 1589 | * Search starts from the last successful isolation order or the next
|
| 1590 | * order to search after a previous failure
|
| 1591 |
|
| 1592 | cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);
|
| 1593 |
|
| 1594 | for (order = cc->search_order;
|
| 1595 | !page && order >= 0;
|
| 1596 | order = next_search_order(cc, order)) {
|
| 1597 | struct free_area *area = &cc->zone->free_area[order];
|
| 1598 | struct list_head *freelist;
|
| 1599 | struct page *freepage;
|
| 1600 | unsigned long flags;
|
| 1601 | unsigned int order_scanned = 0;
|
| 1602 | unsigned long high_pfn = 0;
|
| 1603 |
|
| 1604 | if (!area->nr_free)
|
| 1605 | continue;
|
| 1606 |
|
| 1607 | spin_lock_irqsave(&cc->zone->lock, flags);
|
| 1608 | freelist = &area->free_list[MIGRATE_MOVABLE];
|
| 1609 | list_for_each_entry_reverse(freepage, freelist, buddy_list) {
|
| 1610 | unsigned long pfn;
|
| 1611 |
|
| 1612 | order_scanned++;
|
| 1613 | nr_scanned++;
|
| 1614 | pfn = page_to_pfn(freepage);
|
| 1615 |
|
| 1616 | if (pfn >= highest)
|
| 1617 | highest = max(pageblock_start_pfn(pfn),
|
| 1618 | cc->zone->zone_start_pfn);
|
| 1619 |
|
| 1620 | if (pfn >= low_pfn) {
|
| 1621 | cc->fast_search_fail = 0;
|
| 1622 | cc->search_order = order;
|
| 1623 | page = freepage;
|
| 1624 | break;
|
| 1625 | }
|
| 1626 |
|
| 1627 | if (pfn >= min_pfn && pfn > high_pfn) {
|
| 1628 | high_pfn = pfn;
|
| 1629 |
|
| 1630 |
|
| 1631 | limit >>= 1;
|
| 1632 | }
|
| 1633 |
|
| 1634 | if (order_scanned >= limit)
|
| 1635 | break;
|
| 1636 | }
|
| 1637 |
|
| 1638 |
|
| 1639 | if (!page && high_pfn) {
|
| 1640 | page = pfn_to_page(high_pfn);
|
| 1641 |
|
| 1642 |
|
| 1643 | freepage = page;
|
| 1644 | }
|
| 1645 |
|
| 1646 |
|
| 1647 | move_freelist_head(freelist, freepage);
|
| 1648 |
|
| 1649 |
|
| 1650 | if (page) {
|
| 1651 | if (__isolate_free_page(page, order)) {
|
| 1652 | nr_isolated = 1 << order;
|
| 1653 | nr_scanned += nr_isolated - 1;
|
| 1654 | total_isolated += nr_isolated;
|
| 1655 | cc->nr_freepages += nr_isolated;
|
| 1656 | list_add_tail(&page->lru, &cc->freepages[order]);
|
| 1657 | count_compact_events(COMPACTISOLATED, nr_isolated);
|
| 1658 | } else {
|
| 1659 |
|
| 1660 | order = cc->search_order + 1;
|
| 1661 | page = NULL;
|
| 1662 | }
|
| 1663 | }
|
| 1664 |
|
| 1665 | spin_unlock_irqrestore(&cc->zone->lock, flags);
|
| 1666 |
|
| 1667 |
|
| 1668 | if (cc->nr_freepages >= cc->nr_migratepages)
|
| 1669 | break;
|
| 1670 |
|
| 1671 | |
| 1672 | * Smaller scan on next order so the total scan is related
|
| 1673 | * to freelist_scan_limit.
|
| 1674 |
|
| 1675 | if (order_scanned >= limit)
|
| 1676 | limit = max(1U, limit >> 1);
|
| 1677 | }
|
| 1678 |
|
| 1679 | trace_mm_compaction_fast_isolate_freepages(min_pfn, cc->free_pfn,
|
| 1680 | nr_scanned, total_isolated);
|
| 1681 |
|
| 1682 | if (!page) {
|
| 1683 | cc->fast_search_fail++;
|
| 1684 | if (scan_start) {
|
| 1685 | |
| 1686 | * Use the highest PFN found above min. If one was
|
| 1687 | * not found, be pessimistic for direct compaction
|
| 1688 | * and use the min mark.
|
| 1689 |
|
| 1690 | if (highest >= min_pfn) {
|
| 1691 | page = pfn_to_page(highest);
|
| 1692 | cc->free_pfn = highest;
|
| 1693 | } else {
|
| 1694 | if (cc->direct_compaction && pfn_valid(min_pfn)) {
|
| 1695 | page = pageblock_pfn_to_page(min_pfn,
|
| 1696 | min(pageblock_end_pfn(min_pfn),
|
| 1697 | zone_end_pfn(cc->zone)),
|
| 1698 | cc->zone);
|
| 1699 | if (page && !suitable_migration_target(cc, page))
|
| 1700 | page = NULL;
|
| 1701 |
|
| 1702 | cc->free_pfn = min_pfn;
|
| 1703 | }
|
| 1704 | }
|
| 1705 | }
|
| 1706 | }
|
| 1707 |
|
| 1708 | if (highest && highest >= cc->zone->compact_cached_free_pfn) {
|
| 1709 | highest -= pageblock_nr_pages;
|
| 1710 | cc->zone->compact_cached_free_pfn = highest;
|
| 1711 | }
|
| 1712 |
|
| 1713 | cc->total_free_scanned += nr_scanned;
|
| 1714 | if (!page)
|
| 1715 | return;
|
| 1716 |
|
| 1717 | low_pfn = page_to_pfn(page);
|
| 1718 | fast_isolate_around(cc, low_pfn);
|
| 1719 | }
|
| 1720 |
|
| 1721 | |
| 1722 | * Based on information in the current compact_control, find blocks
|
| 1723 | * suitable for isolating free pages from and then isolate them.
|
| 1724 |
|
| 1725 | static void isolate_freepages(struct compact_control *cc)
|
| 1726 | {
|
| 1727 | struct zone *zone = cc->zone;
|
| 1728 | struct page *page;
|
| 1729 | unsigned long block_start_pfn;
|
| 1730 | unsigned long isolate_start_pfn;
|
| 1731 | unsigned long block_end_pfn;
|
| 1732 | unsigned long low_pfn;
|
| 1733 | unsigned int stride;
|
| 1734 |
|
| 1735 |
|
| 1736 | fast_isolate_freepages(cc);
|
| 1737 | if (cc->nr_freepages)
|
| 1738 | return;
|
| 1739 |
|
| 1740 | |
| 1741 | * Initialise the free scanner. The starting point is where we last
|
| 1742 | * successfully isolated from, zone-cached value, or the end of the
|
| 1743 | * zone when isolating for the first time. For looping we also need
|
| 1744 | * this pfn aligned down to the pageblock boundary, because we do
|
| 1745 | * block_start_pfn -= pageblock_nr_pages in the for loop.
|
| 1746 | * For ending point, take care when isolating in last pageblock of a
|
| 1747 | * zone which ends in the middle of a pageblock.
|
| 1748 | * The low boundary is the end of the pageblock the migration scanner
|
| 1749 | * is using.
|
| 1750 |
|
| 1751 | isolate_start_pfn = cc->free_pfn;
|
| 1752 | block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
|
| 1753 | block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
|
| 1754 | zone_end_pfn(zone));
|
| 1755 | low_pfn = pageblock_end_pfn(cc->migrate_pfn);
|
| 1756 | stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
|
| 1757 |
|
| 1758 | |
| 1759 | * Isolate free pages until enough are available to migrate the
|
| 1760 | * pages on cc->migratepages. We stop searching if the migrate
|
| 1761 | * and free page scanners meet or enough free pages are isolated.
|
| 1762 |
|
| 1763 | for (; block_start_pfn >= low_pfn;
|
| 1764 | block_end_pfn = block_start_pfn,
|
| 1765 | block_start_pfn -= pageblock_nr_pages,
|
| 1766 | isolate_start_pfn = block_start_pfn) {
|
| 1767 | unsigned long nr_isolated;
|
| 1768 |
|
| 1769 | |
| 1770 | * This can iterate a massively long zone without finding any
|
| 1771 | * suitable migration targets, so periodically check resched.
|
| 1772 |
|
| 1773 | if (!(block_start_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
|
| 1774 | cond_resched();
|
| 1775 |
|
| 1776 | page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
|
| 1777 | zone);
|
| 1778 | if (!page) {
|
| 1779 | unsigned long next_pfn;
|
| 1780 |
|
| 1781 | next_pfn = skip_offline_sections_reverse(block_start_pfn);
|
| 1782 | if (next_pfn)
|
| 1783 | block_start_pfn = max(next_pfn, low_pfn);
|
| 1784 |
|
| 1785 | continue;
|
| 1786 | }
|
| 1787 |
|
| 1788 |
|
| 1789 | if (!suitable_migration_target(cc, page))
|
| 1790 | continue;
|
| 1791 |
|
| 1792 |
|
| 1793 | if (!isolation_suitable(cc, page))
|
| 1794 | continue;
|
| 1795 |
|
| 1796 |
|
| 1797 | nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
|
| 1798 | block_end_pfn, cc->freepages, stride, false);
|
| 1799 |
|
| 1800 |
|
| 1801 | if (isolate_start_pfn == block_end_pfn)
|
| 1802 | update_pageblock_skip(cc, page, block_start_pfn -
|
| 1803 | pageblock_nr_pages);
|
| 1804 |
|
| 1805 |
|
| 1806 | if (cc->nr_freepages >= cc->nr_migratepages) {
|
| 1807 | if (isolate_start_pfn >= block_end_pfn) {
|
| 1808 | |
| 1809 | * Restart at previous pageblock if more
|
| 1810 | * freepages can be isolated next time.
|
| 1811 |
|
| 1812 | isolate_start_pfn =
|
| 1813 | block_start_pfn - pageblock_nr_pages;
|
| 1814 | }
|
| 1815 | break;
|
| 1816 | } else if (isolate_start_pfn < block_end_pfn) {
|
| 1817 | |
| 1818 | * If isolation failed early, do not continue
|
| 1819 | * needlessly.
|
| 1820 |
|
| 1821 | break;
|
| 1822 | }
|
| 1823 |
|
| 1824 |
|
| 1825 | if (nr_isolated) {
|
| 1826 | stride = 1;
|
| 1827 | continue;
|
| 1828 | }
|
| 1829 | stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
|
| 1830 | }
|
| 1831 |
|
| 1832 | |
| 1833 | * Record where the free scanner will restart next time. Either we
|
| 1834 | * broke from the loop and set isolate_start_pfn based on the last
|
| 1835 | * call to isolate_freepages_block(), or we met the migration scanner
|
| 1836 | * and the loop terminated due to isolate_start_pfn < low_pfn
|
| 1837 |
|
| 1838 | cc->free_pfn = isolate_start_pfn;
|
| 1839 | }
|
| 1840 |
|
| 1841 | |
| 1842 | * This is a migrate-callback that "allocates" freepages by taking pages
|
| 1843 | * from the isolated freelists in the block we are migrating to.
|
| 1844 |
|
| 1845 | static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long data)
|
| 1846 | {
|
| 1847 | struct compact_control *cc = (struct compact_control *)data;
|
| 1848 | struct folio *dst;
|
| 1849 | int order = folio_order(src);
|
| 1850 | bool has_isolated_pages = false;
|
| 1851 | int start_order;
|
| 1852 | struct page *freepage;
|
| 1853 | unsigned long size;
|
| 1854 |
|
| 1855 | again:
|
| 1856 | for (start_order = order; start_order < NR_PAGE_ORDERS; start_order++)
|
| 1857 | if (!list_empty(&cc->freepages[start_order]))
|
| 1858 | break;
|
| 1859 |
|
| 1860 |
|
| 1861 | if (start_order == NR_PAGE_ORDERS) {
|
| 1862 | if (has_isolated_pages)
|
| 1863 | return NULL;
|
| 1864 | isolate_freepages(cc);
|
| 1865 | has_isolated_pages = true;
|
| 1866 | goto again;
|
| 1867 | }
|
| 1868 |
|
| 1869 | freepage = list_first_entry(&cc->freepages[start_order], struct page,
|
| 1870 | lru);
|
| 1871 | size = 1 << start_order;
|
| 1872 |
|
| 1873 | list_del(&freepage->lru);
|
| 1874 |
|
| 1875 | while (start_order > order) {
|
| 1876 | start_order--;
|
| 1877 | size >>= 1;
|
| 1878 |
|
| 1879 | list_add(&freepage[size].lru, &cc->freepages[start_order]);
|
| 1880 | }
|
| 1881 | dst = (struct folio *)freepage;
|
| 1882 |
|
| 1883 | post_alloc_hook(&dst->page, order, __GFP_MOVABLE, ALLOC_DEFAULT);
|
| 1884 | set_page_refcounted(&dst->page);
|
| 1885 | if (order)
|
| 1886 | prep_compound_page(&dst->page, order);
|
| 1887 | cc->nr_freepages -= 1 << order;
|
| 1888 | cc->nr_migratepages -= 1 << order;
|
| 1889 | return page_rmappable_folio(&dst->page);
|
| 1890 | }
|
| 1891 |
|
| 1892 | static struct folio *compaction_alloc(struct folio *src, unsigned long data)
|
| 1893 | {
|
| 1894 | return alloc_hooks(compaction_alloc_noprof(src, data));
|
| 1895 | }
|
| 1896 |
|
| 1897 | |
| 1898 | * This is a migrate-callback that "frees" freepages back to the isolated
|
| 1899 | * freelist. All pages on the freelist are from the same zone, so there is no
|
| 1900 | * special handling needed for NUMA.
|
| 1901 |
|
| 1902 | static void compaction_free(struct folio *dst, unsigned long data)
|
| 1903 | {
|
| 1904 | struct compact_control *cc = (struct compact_control *)data;
|
| 1905 | int order = folio_order(dst);
|
| 1906 | struct page *page = &dst->page;
|
| 1907 |
|
| 1908 | if (folio_put_testzero(dst) && free_pages_prepare(page, order)) {
|
| 1909 | list_add(&dst->lru, &cc->freepages[order]);
|
| 1910 | cc->nr_freepages += 1 << order;
|
| 1911 | }
|
| 1912 | cc->nr_migratepages += 1 << order;
|
| 1913 | |
| 1914 | * someone else has referenced the page or free_pages_prepare() fails,
|
| 1915 | * we cannot take it back to our free list.
|
| 1916 |
|
| 1917 | }
|
| 1918 |
|
| 1919 |
|
| 1920 | typedef enum {
|
| 1921 | ISOLATE_ABORT,
|
| 1922 | ISOLATE_NONE,
|
| 1923 | ISOLATE_SUCCESS,
|
| 1924 | } isolate_migrate_t;
|
| 1925 |
|
| 1926 | |
| 1927 | * Allow userspace to control policy on scanning the unevictable LRU for
|
| 1928 | * compactable pages.
|
| 1929 |
|
| 1930 | static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT;
|
| 1931 | |
| 1932 | * Tunable for proactive compaction. It determines how
|
| 1933 | * aggressively the kernel should compact memory in the
|
| 1934 | * background. It takes values in the range [0, 100].
|
| 1935 |
|
| 1936 | static unsigned int __read_mostly sysctl_compaction_proactiveness = 20;
|
| 1937 | static int sysctl_extfrag_threshold = 500;
|
| 1938 | static int __read_mostly sysctl_compact_memory;
|
| 1939 |
|
| 1940 | static inline void
|
| 1941 | update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
|
| 1942 | {
|
| 1943 | if (cc->fast_start_pfn == ULONG_MAX)
|
| 1944 | return;
|
| 1945 |
|
| 1946 | if (!cc->fast_start_pfn)
|
| 1947 | cc->fast_start_pfn = pfn;
|
| 1948 |
|
| 1949 | cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
|
| 1950 | }
|
| 1951 |
|
| 1952 | static inline unsigned long
|
| 1953 | reinit_migrate_pfn(struct compact_control *cc)
|
| 1954 | {
|
| 1955 | if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
|
| 1956 | return cc->migrate_pfn;
|
| 1957 |
|
| 1958 | cc->migrate_pfn = cc->fast_start_pfn;
|
| 1959 | cc->fast_start_pfn = ULONG_MAX;
|
| 1960 |
|
| 1961 | return cc->migrate_pfn;
|
| 1962 | }
|
| 1963 |
|
| 1964 | |
| 1965 | * Briefly search the free lists for a migration source that already has
|
| 1966 | * some free pages to reduce the number of pages that need migration
|
| 1967 | * before a pageblock is free.
|
| 1968 |
|
| 1969 | static unsigned long fast_find_migrateblock(struct compact_control *cc)
|
| 1970 | {
|
| 1971 | unsigned int limit = freelist_scan_limit(cc);
|
| 1972 | unsigned int nr_scanned = 0;
|
| 1973 | unsigned long distance;
|
| 1974 | unsigned long pfn = cc->migrate_pfn;
|
| 1975 | unsigned long high_pfn;
|
| 1976 | int order;
|
| 1977 | bool found_block = false;
|
| 1978 |
|
| 1979 |
|
| 1980 | if (cc->ignore_skip_hint)
|
| 1981 | return pfn;
|
| 1982 |
|
| 1983 | |
| 1984 | * If the pageblock should be finished then do not select a different
|
| 1985 | * pageblock.
|
| 1986 |
|
| 1987 | if (cc->finish_pageblock)
|
| 1988 | return pfn;
|
| 1989 |
|
| 1990 | |
| 1991 | * If the migrate_pfn is not at the start of a zone or the start
|
| 1992 | * of a pageblock then assume this is a continuation of a previous
|
| 1993 | * scan restarted due to COMPACT_CLUSTER_MAX.
|
| 1994 |
|
| 1995 | if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
|
| 1996 | return pfn;
|
| 1997 |
|
| 1998 | |
| 1999 | * For smaller orders, just linearly scan as the number of pages
|
| 2000 | * to migrate should be relatively small and does not necessarily
|
| 2001 | * justify freeing up a large block for a small allocation.
|
| 2002 |
|
| 2003 | if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
|
| 2004 | return pfn;
|
| 2005 |
|
| 2006 | |
| 2007 | * Prevent <pageblock_order unmovable/reclaimable requests from
|
| 2008 | * polluting movable blocks through fallbacks. Whole-block production
|
| 2009 | * is exempt as the allocator claims and converts these.
|
| 2010 |
|
| 2011 | if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE &&
|
| 2012 | cc->order < pageblock_order)
|
| 2013 | return pfn;
|
| 2014 |
|
| 2015 | |
| 2016 | * When starting the migration scanner, pick any pageblock within the
|
| 2017 | * first half of the search space. Otherwise try and pick a pageblock
|
| 2018 | * within the first eighth to reduce the chances that a migration
|
| 2019 | * target later becomes a source.
|
| 2020 |
|
| 2021 | distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
|
| 2022 | if (cc->migrate_pfn != cc->zone->zone_start_pfn)
|
| 2023 | distance >>= 2;
|
| 2024 | high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);
|
| 2025 |
|
| 2026 | for (order = cc->order - 1;
|
| 2027 | order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;
|
| 2028 | order--) {
|
| 2029 | struct free_area *area = &cc->zone->free_area[order];
|
| 2030 | struct list_head *freelist;
|
| 2031 | unsigned long flags;
|
| 2032 | struct page *freepage;
|
| 2033 |
|
| 2034 | if (!area->nr_free)
|
| 2035 | continue;
|
| 2036 |
|
| 2037 | spin_lock_irqsave(&cc->zone->lock, flags);
|
| 2038 | freelist = &area->free_list[MIGRATE_MOVABLE];
|
| 2039 | list_for_each_entry(freepage, freelist, buddy_list) {
|
| 2040 | unsigned long free_pfn;
|
| 2041 |
|
| 2042 | if (nr_scanned++ >= limit) {
|
| 2043 | move_freelist_tail(freelist, freepage);
|
| 2044 | break;
|
| 2045 | }
|
| 2046 |
|
| 2047 | free_pfn = page_to_pfn(freepage);
|
| 2048 | if (free_pfn < high_pfn) {
|
| 2049 | |
| 2050 | * Avoid if skipped recently. Ideally it would
|
| 2051 | * move to the tail but even safe iteration of
|
| 2052 | * the list assumes an entry is deleted, not
|
| 2053 | * reordered.
|
| 2054 |
|
| 2055 | if (get_pageblock_skip(freepage))
|
| 2056 | continue;
|
| 2057 |
|
| 2058 |
|
| 2059 | move_freelist_tail(freelist, freepage);
|
| 2060 |
|
| 2061 | update_fast_start_pfn(cc, free_pfn);
|
| 2062 | pfn = pageblock_start_pfn(free_pfn);
|
| 2063 | if (pfn < cc->zone->zone_start_pfn)
|
| 2064 | pfn = cc->zone->zone_start_pfn;
|
| 2065 | cc->fast_search_fail = 0;
|
| 2066 | found_block = true;
|
| 2067 | break;
|
| 2068 | }
|
| 2069 | }
|
| 2070 | spin_unlock_irqrestore(&cc->zone->lock, flags);
|
| 2071 | }
|
| 2072 |
|
| 2073 | cc->total_migrate_scanned += nr_scanned;
|
| 2074 |
|
| 2075 | |
| 2076 | * If fast scanning failed then use a cached entry for a page block
|
| 2077 | * that had free pages as the basis for starting a linear scan.
|
| 2078 |
|
| 2079 | if (!found_block) {
|
| 2080 | cc->fast_search_fail++;
|
| 2081 | pfn = reinit_migrate_pfn(cc);
|
| 2082 | }
|
| 2083 | return pfn;
|
| 2084 | }
|
| 2085 |
|
| 2086 | |
| 2087 | * Isolate all pages that can be migrated from the first suitable block,
|
| 2088 | * starting at the block pointed to by the migrate scanner pfn within
|
| 2089 | * compact_control.
|
| 2090 |
|
| 2091 | static isolate_migrate_t isolate_migratepages(struct compact_control *cc)
|
| 2092 | {
|
| 2093 | unsigned long block_start_pfn;
|
| 2094 | unsigned long block_end_pfn;
|
| 2095 | unsigned long low_pfn;
|
| 2096 | struct page *page;
|
| 2097 | const isolate_mode_t isolate_mode =
|
| 2098 | (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
|
| 2099 | (cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
|
| 2100 | bool fast_find_block;
|
| 2101 |
|
| 2102 | |
| 2103 | * Start at where we last stopped, or beginning of the zone as
|
| 2104 | * initialized by compact_zone(). The first failure will use
|
| 2105 | * the lowest PFN as the starting point for linear scanning.
|
| 2106 |
|
| 2107 | low_pfn = fast_find_migrateblock(cc);
|
| 2108 | block_start_pfn = pageblock_start_pfn(low_pfn);
|
| 2109 | if (block_start_pfn < cc->zone->zone_start_pfn)
|
| 2110 | block_start_pfn = cc->zone->zone_start_pfn;
|
| 2111 |
|
| 2112 | |
| 2113 | * fast_find_migrateblock() has already ensured the pageblock is not
|
| 2114 | * set with a skipped flag, so to avoid the isolation_suitable check
|
| 2115 | * below again, check whether the fast search was successful.
|
| 2116 |
|
| 2117 | fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;
|
| 2118 |
|
| 2119 |
|
| 2120 | block_end_pfn = pageblock_end_pfn(low_pfn);
|
| 2121 |
|
| 2122 | |
| 2123 | * Iterate over whole pageblocks until we find the first suitable.
|
| 2124 | * Do not cross the free scanner.
|
| 2125 |
|
| 2126 | for (; block_end_pfn <= cc->free_pfn;
|
| 2127 | fast_find_block = false,
|
| 2128 | cc->migrate_pfn = low_pfn = block_end_pfn,
|
| 2129 | block_start_pfn = block_end_pfn,
|
| 2130 | block_end_pfn += pageblock_nr_pages) {
|
| 2131 |
|
| 2132 | |
| 2133 | * This can potentially iterate a massively long zone with
|
| 2134 | * many pageblocks unsuitable, so periodically check if we
|
| 2135 | * need to schedule.
|
| 2136 |
|
| 2137 | if (!(low_pfn % (COMPACT_CLUSTER_MAX * pageblock_nr_pages)))
|
| 2138 | cond_resched();
|
| 2139 |
|
| 2140 | page = pageblock_pfn_to_page(block_start_pfn,
|
| 2141 | block_end_pfn, cc->zone);
|
| 2142 | if (!page) {
|
| 2143 | unsigned long next_pfn;
|
| 2144 |
|
| 2145 | next_pfn = skip_offline_sections(block_start_pfn);
|
| 2146 | if (next_pfn)
|
| 2147 | block_end_pfn = min(next_pfn, cc->free_pfn);
|
| 2148 | continue;
|
| 2149 | }
|
| 2150 |
|
| 2151 | |
| 2152 | * If isolation recently failed, do not retry. Only check the
|
| 2153 | * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock
|
| 2154 | * to be visited multiple times. Assume skip was checked
|
| 2155 | * before making it "skip" so other compaction instances do
|
| 2156 | * not scan the same block.
|
| 2157 |
|
| 2158 | if ((pageblock_aligned(low_pfn) ||
|
| 2159 | low_pfn == cc->zone->zone_start_pfn) &&
|
| 2160 | !fast_find_block && !isolation_suitable(cc, page))
|
| 2161 | continue;
|
| 2162 |
|
| 2163 | |
| 2164 | * For async direct compaction, only scan the pageblocks of the
|
| 2165 | * same migratetype without huge pages. Async direct compaction
|
| 2166 | * is optimistic to see if the minimum amount of work satisfies
|
| 2167 | * the allocation. The cached PFN is updated as it's possible
|
| 2168 | * that all remaining blocks between source and target are
|
| 2169 | * unsuitable and the compaction scanners fail to meet.
|
| 2170 |
|
| 2171 | if (!suitable_migration_source(cc, page)) {
|
| 2172 | update_cached_migrate(cc, block_end_pfn);
|
| 2173 | continue;
|
| 2174 | }
|
| 2175 |
|
| 2176 |
|
| 2177 | if (isolate_migratepages_block(cc, low_pfn, block_end_pfn,
|
| 2178 | isolate_mode))
|
| 2179 | return ISOLATE_ABORT;
|
| 2180 |
|
| 2181 | |
| 2182 | * Either we isolated something and proceed with migration. Or
|
| 2183 | * we failed and compact_zone should decide if we should
|
| 2184 | * continue or not.
|
| 2185 |
|
| 2186 | break;
|
| 2187 | }
|
| 2188 |
|
| 2189 | return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
|
| 2190 | }
|
| 2191 |
|
| 2192 | |
| 2193 | * Determine whether kswapd is (or recently was!) running on this node.
|
| 2194 | *
|
| 2195 | * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't
|
| 2196 | * zero it.
|
| 2197 |
|
| 2198 | static bool kswapd_is_running(pg_data_t *pgdat)
|
| 2199 | {
|
| 2200 | bool running;
|
| 2201 |
|
| 2202 | pgdat_kswapd_lock(pgdat);
|
| 2203 | running = pgdat->kswapd && task_is_running(pgdat->kswapd);
|
| 2204 | pgdat_kswapd_unlock(pgdat);
|
| 2205 |
|
| 2206 | return running;
|
| 2207 | }
|
| 2208 |
|
| 2209 | |
| 2210 | * A zone's fragmentation score is the external fragmentation wrt to the
|
| 2211 | * COMPACTION_HPAGE_ORDER. It returns a value in the range [0, 100].
|
| 2212 |
|
| 2213 | static unsigned int fragmentation_score_zone(struct zone *zone)
|
| 2214 | {
|
| 2215 | return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);
|
| 2216 | }
|
| 2217 |
|
| 2218 | |
| 2219 | * A weighted zone's fragmentation score is the external fragmentation
|
| 2220 | * wrt to the COMPACTION_HPAGE_ORDER scaled by the zone's size. It
|
| 2221 | * returns a value in the range [0, 100].
|
| 2222 | *
|
| 2223 | * The scaling factor ensures that proactive compaction focuses on larger
|
| 2224 | * zones like ZONE_NORMAL, rather than smaller, specialized zones like
|
| 2225 | * ZONE_DMA32. For smaller zones, the score value remains close to zero,
|
| 2226 | * and thus never exceeds the high threshold for proactive compaction.
|
| 2227 |
|
| 2228 | static unsigned int fragmentation_score_zone_weighted(struct zone *zone)
|
| 2229 | {
|
| 2230 | unsigned long score;
|
| 2231 |
|
| 2232 | score = zone->present_pages * fragmentation_score_zone(zone);
|
| 2233 | return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);
|
| 2234 | }
|
| 2235 |
|
| 2236 | |
| 2237 | * The per-node proactive (background) compaction process is started by its
|
| 2238 | * corresponding kcompactd thread when the node's fragmentation score
|
| 2239 | * exceeds the high threshold. The compaction process remains active till
|
| 2240 | * the node's score falls below the low threshold, or one of the back-off
|
| 2241 | * conditions is met.
|
| 2242 |
|
| 2243 | static unsigned int fragmentation_score_node(pg_data_t *pgdat)
|
| 2244 | {
|
| 2245 | unsigned int score = 0;
|
| 2246 | int zoneid;
|
| 2247 |
|
| 2248 | for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
|
| 2249 | struct zone *zone;
|
| 2250 |
|
| 2251 | zone = &pgdat->node_zones[zoneid];
|
| 2252 | if (!populated_zone(zone))
|
| 2253 | continue;
|
| 2254 | score += fragmentation_score_zone_weighted(zone);
|
| 2255 | }
|
| 2256 |
|
| 2257 | return score;
|
| 2258 | }
|
| 2259 |
|
| 2260 | static unsigned int fragmentation_score_wmark(bool low)
|
| 2261 | {
|
| 2262 | unsigned int wmark_low, leeway;
|
| 2263 |
|
| 2264 | wmark_low = 100U - sysctl_compaction_proactiveness;
|
| 2265 | leeway = min(10U, wmark_low / 2);
|
| 2266 | return low ? wmark_low : min(wmark_low + leeway, 100U);
|
| 2267 | }
|
| 2268 |
|
| 2269 | static bool should_proactive_compact_node(pg_data_t *pgdat)
|
| 2270 | {
|
| 2271 | int wmark_high;
|
| 2272 |
|
| 2273 | if (!sysctl_compaction_proactiveness || kswapd_is_running(pgdat))
|
| 2274 | return false;
|
| 2275 |
|
| 2276 | wmark_high = fragmentation_score_wmark(false);
|
| 2277 | return fragmentation_score_node(pgdat) > wmark_high;
|
| 2278 | }
|
| 2279 |
|
| 2280 | static enum compact_result __compact_finished(struct compact_control *cc)
|
| 2281 | {
|
| 2282 | unsigned int order;
|
| 2283 | const int migratetype = cc->migratetype;
|
| 2284 | int ret;
|
| 2285 |
|
| 2286 |
|
| 2287 | if (compact_scanners_met(cc)) {
|
| 2288 |
|
| 2289 | reset_cached_positions(cc->zone);
|
| 2290 |
|
| 2291 | |
| 2292 | * Mark that the PG_migrate_skip information should be cleared
|
| 2293 | * by kswapd when it goes to sleep. kcompactd does not set the
|
| 2294 | * flag itself as the decision to be clear should be directly
|
| 2295 | * based on an allocation request.
|
| 2296 |
|
| 2297 | if (cc->direct_compaction)
|
| 2298 | cc->zone->compact_blockskip_flush = true;
|
| 2299 |
|
| 2300 | if (cc->whole_zone)
|
| 2301 | return COMPACT_COMPLETE;
|
| 2302 | else
|
| 2303 | return COMPACT_PARTIAL_SKIPPED;
|
| 2304 | }
|
| 2305 |
|
| 2306 | if (cc->proactive_compaction) {
|
| 2307 | int score, wmark_low;
|
| 2308 | pg_data_t *pgdat;
|
| 2309 |
|
| 2310 | pgdat = cc->zone->zone_pgdat;
|
| 2311 | if (kswapd_is_running(pgdat))
|
| 2312 | return COMPACT_PARTIAL_SKIPPED;
|
| 2313 |
|
| 2314 | score = fragmentation_score_zone(cc->zone);
|
| 2315 | wmark_low = fragmentation_score_wmark(true);
|
| 2316 |
|
| 2317 | if (score > wmark_low)
|
| 2318 | ret = COMPACT_CONTINUE;
|
| 2319 | else
|
| 2320 | ret = COMPACT_SUCCESS;
|
| 2321 |
|
| 2322 | goto out;
|
| 2323 | }
|
| 2324 |
|
| 2325 | if (is_via_compact_memory(cc->order))
|
| 2326 | return COMPACT_CONTINUE;
|
| 2327 |
|
| 2328 | |
| 2329 | * Always finish scanning a pageblock to reduce the possibility of
|
| 2330 | * fallbacks in the future. This is particularly important when
|
| 2331 | * migration source is unmovable/reclaimable but it's not worth
|
| 2332 | * special casing.
|
| 2333 |
|
| 2334 | if (!pageblock_aligned(cc->migrate_pfn))
|
| 2335 | return COMPACT_CONTINUE;
|
| 2336 |
|
| 2337 | |
| 2338 | * When defrag_mode is enabled, make kcompactd target
|
| 2339 | * watermarks in whole pageblocks. Because they can be stolen
|
| 2340 | * without polluting, no further fallback checks are needed.
|
| 2341 |
|
| 2342 | if (defrag_mode && !cc->direct_compaction) {
|
| 2343 | if (__zone_watermark_ok(cc->zone, cc->order,
|
| 2344 | high_wmark_pages(cc->zone),
|
| 2345 | cc->highest_zoneidx, cc->alloc_flags,
|
| 2346 | zone_page_state(cc->zone,
|
| 2347 | NR_FREE_PAGES_BLOCKS)))
|
| 2348 | return COMPACT_SUCCESS;
|
| 2349 |
|
| 2350 | return COMPACT_CONTINUE;
|
| 2351 | }
|
| 2352 |
|
| 2353 |
|
| 2354 | ret = COMPACT_NO_SUITABLE_PAGE;
|
| 2355 | for (order = cc->order; order < NR_PAGE_ORDERS; order++) {
|
| 2356 | struct free_area *area = &cc->zone->free_area[order];
|
| 2357 |
|
| 2358 |
|
| 2359 | if (!free_area_empty(area, migratetype))
|
| 2360 | return COMPACT_SUCCESS;
|
| 2361 |
|
| 2362 | #ifdef CONFIG_CMA
|
| 2363 |
|
| 2364 | if (migratetype == MIGRATE_MOVABLE &&
|
| 2365 | !free_area_empty(area, MIGRATE_CMA))
|
| 2366 | return COMPACT_SUCCESS;
|
| 2367 | #endif
|
| 2368 | |
| 2369 | * Job done if allocation would steal freepages from
|
| 2370 | * other migratetype buddy lists.
|
| 2371 |
|
| 2372 | if (find_suitable_fallback(area, order, migratetype, true, NULL)
|
| 2373 | == FALLBACK_FOUND)
|
| 2374 | |
| 2375 | * Movable pages are OK in any pageblock. If we are
|
| 2376 | * stealing for a non-movable allocation, make sure
|
| 2377 | * we finish compacting the current pageblock first
|
| 2378 | * (which is assured by the above migrate_pfn align
|
| 2379 | * check) so it is as free as possible and we won't
|
| 2380 | * have to steal another one soon.
|
| 2381 |
|
| 2382 | return COMPACT_SUCCESS;
|
| 2383 | }
|
| 2384 |
|
| 2385 | out:
|
| 2386 | if (cc->contended || fatal_signal_pending(current))
|
| 2387 | ret = COMPACT_CONTENDED;
|
| 2388 |
|
| 2389 | return ret;
|
| 2390 | }
|
| 2391 |
|
| 2392 | static enum compact_result compact_finished(struct compact_control *cc)
|
| 2393 | {
|
| 2394 | int ret;
|
| 2395 |
|
| 2396 | ret = __compact_finished(cc);
|
| 2397 | trace_mm_compaction_finished(cc->zone, cc->order, ret);
|
| 2398 | if (ret == COMPACT_NO_SUITABLE_PAGE)
|
| 2399 | ret = COMPACT_CONTINUE;
|
| 2400 |
|
| 2401 | return ret;
|
| 2402 | }
|
| 2403 |
|
| 2404 | static bool __compaction_suitable(struct zone *zone, int order,
|
| 2405 | unsigned long watermark, int highest_zoneidx,
|
| 2406 | unsigned long free_pages)
|
| 2407 | {
|
| 2408 | |
| 2409 | * Watermarks for order-0 must be met for compaction to be able to
|
| 2410 | * isolate free pages for migration targets. This means that the
|
| 2411 | * watermark have to match, or be more pessimistic than the check in
|
| 2412 | * __isolate_free_page().
|
| 2413 | *
|
| 2414 | * For costly orders, we require a higher watermark for compaction to
|
| 2415 | * proceed to increase its chances.
|
| 2416 | *
|
| 2417 | * We use the direct compactor's highest_zoneidx to skip over zones
|
| 2418 | * where lowmem reserves would prevent allocation even if compaction
|
| 2419 | * succeeds.
|
| 2420 | *
|
| 2421 | * ALLOC_CMA is used, as pages in CMA pageblocks are considered
|
| 2422 | * suitable migration targets.
|
| 2423 |
|
| 2424 | watermark += compact_gap(order);
|
| 2425 | if (order > PAGE_ALLOC_COSTLY_ORDER)
|
| 2426 | watermark += low_wmark_pages(zone) - min_wmark_pages(zone);
|
| 2427 | return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,
|
| 2428 | ALLOC_CMA, free_pages);
|
| 2429 | }
|
| 2430 |
|
| 2431 | |
| 2432 | * compaction_suitable: Is this suitable to run compaction on this zone now?
|
| 2433 |
|
| 2434 | bool compaction_suitable(struct zone *zone, int order, unsigned long watermark,
|
| 2435 | int highest_zoneidx)
|
| 2436 | {
|
| 2437 | enum compact_result compact_result;
|
| 2438 | bool suitable;
|
| 2439 |
|
| 2440 | suitable = __compaction_suitable(zone, order, watermark, highest_zoneidx,
|
| 2441 | zone_page_state(zone, NR_FREE_PAGES));
|
| 2442 | |
| 2443 | * fragmentation index determines if allocation failures are due to
|
| 2444 | * low memory or external fragmentation
|
| 2445 | *
|
| 2446 | * index of -1000 would imply allocations might succeed depending on
|
| 2447 | * watermarks, but we already failed the high-order watermark check
|
| 2448 | * index towards 0 implies failure is due to lack of memory
|
| 2449 | * index towards 1000 implies failure is due to fragmentation
|
| 2450 | *
|
| 2451 | * Only compact if a failure would be due to fragmentation. Also
|
| 2452 | * ignore fragindex for non-costly orders where the alternative to
|
| 2453 | * a successful reclaim/compaction is OOM. Fragindex and the
|
| 2454 | * vm.extfrag_threshold sysctl is meant as a heuristic to prevent
|
| 2455 | * excessive compaction for costly orders, but it should not be at the
|
| 2456 | * expense of system stability.
|
| 2457 |
|
| 2458 | if (suitable) {
|
| 2459 | compact_result = COMPACT_CONTINUE;
|
| 2460 | if (order > PAGE_ALLOC_COSTLY_ORDER) {
|
| 2461 | int fragindex = fragmentation_index(zone, order);
|
| 2462 |
|
| 2463 | if (fragindex >= 0 &&
|
| 2464 | fragindex <= sysctl_extfrag_threshold) {
|
| 2465 | suitable = false;
|
| 2466 | compact_result = COMPACT_NOT_SUITABLE_ZONE;
|
| 2467 | }
|
| 2468 | }
|
| 2469 | } else {
|
| 2470 | compact_result = COMPACT_SKIPPED;
|
| 2471 | }
|
| 2472 |
|
| 2473 | trace_mm_compaction_suitable(zone, order, compact_result);
|
| 2474 |
|
| 2475 | return suitable;
|
| 2476 | }
|
| 2477 |
|
| 2478 |
|
| 2479 | bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
|
| 2480 | int alloc_flags, gfp_t gfp_mask)
|
| 2481 | {
|
| 2482 | struct zone *zone;
|
| 2483 | struct zoneref *z;
|
| 2484 |
|
| 2485 | |
| 2486 | * Make sure at least one zone would pass __compaction_suitable if we continue
|
| 2487 | * retrying the reclaim.
|
| 2488 |
|
| 2489 | for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
|
| 2490 | ac->highest_zoneidx, ac->nodemask) {
|
| 2491 | unsigned long available;
|
| 2492 |
|
| 2493 | if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
|
| 2494 | !__cpuset_zone_allowed(zone, gfp_mask))
|
| 2495 | continue;
|
| 2496 |
|
| 2497 | |
| 2498 | * Do not consider all the reclaimable memory because we do not
|
| 2499 | * want to trash just for a single high order allocation which
|
| 2500 | * is even not guaranteed to appear even if __compaction_suitable
|
| 2501 | * is happy about the watermark check.
|
| 2502 |
|
| 2503 | available = zone_reclaimable_pages(zone) / order;
|
| 2504 | available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
|
| 2505 | if (__compaction_suitable(zone, order, min_wmark_pages(zone),
|
| 2506 | ac->highest_zoneidx, available))
|
| 2507 | return true;
|
| 2508 | }
|
| 2509 |
|
| 2510 | return false;
|
| 2511 | }
|
| 2512 |
|
| 2513 | |
| 2514 | * Should we do compaction for target allocation order.
|
| 2515 | * Return COMPACT_SUCCESS if allocation for target order can be already
|
| 2516 | * satisfied
|
| 2517 | * Return COMPACT_SKIPPED if compaction for target order is likely to fail
|
| 2518 | * Return COMPACT_CONTINUE if compaction for target order should be ran
|
| 2519 |
|
| 2520 | static enum compact_result
|
| 2521 | compaction_suit_allocation_order(struct zone *zone, unsigned int order,
|
| 2522 | int highest_zoneidx, unsigned int alloc_flags,
|
| 2523 | bool async, bool kcompactd)
|
| 2524 | {
|
| 2525 | unsigned long free_pages;
|
| 2526 | unsigned long watermark;
|
| 2527 |
|
| 2528 | if (kcompactd && defrag_mode)
|
| 2529 | free_pages = zone_page_state(zone, NR_FREE_PAGES_BLOCKS);
|
| 2530 | else
|
| 2531 | free_pages = zone_page_state(zone, NR_FREE_PAGES);
|
| 2532 |
|
| 2533 | watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
|
| 2534 | if (__zone_watermark_ok(zone, order, watermark, highest_zoneidx,
|
| 2535 | alloc_flags, free_pages))
|
| 2536 | return COMPACT_SUCCESS;
|
| 2537 |
|
| 2538 | |
| 2539 | * For unmovable allocations (without ALLOC_CMA), check if there is enough
|
| 2540 | * free memory in the non-CMA pageblocks. Otherwise compaction could form
|
| 2541 | * the high-order page in CMA pageblocks, which would not help the
|
| 2542 | * allocation to succeed. However, limit the check to costly order async
|
| 2543 | * compaction (such as opportunistic THP attempts) because there is the
|
| 2544 | * possibility that compaction would migrate pages from non-CMA to CMA
|
| 2545 | * pageblock.
|
| 2546 |
|
| 2547 | if (order > PAGE_ALLOC_COSTLY_ORDER && async &&
|
| 2548 | !(alloc_flags & ALLOC_CMA)) {
|
| 2549 | if (!__zone_watermark_ok(zone, 0, watermark + compact_gap(order),
|
| 2550 | highest_zoneidx, 0,
|
| 2551 | zone_page_state(zone, NR_FREE_PAGES)))
|
| 2552 | return COMPACT_SKIPPED;
|
| 2553 | }
|
| 2554 |
|
| 2555 | if (!compaction_suitable(zone, order, watermark, highest_zoneidx))
|
| 2556 | return COMPACT_SKIPPED;
|
| 2557 |
|
| 2558 | return COMPACT_CONTINUE;
|
| 2559 | }
|
| 2560 |
|
| 2561 | static enum compact_result
|
| 2562 | compact_zone(struct compact_control *cc, struct capture_control *capc)
|
| 2563 | {
|
| 2564 | enum compact_result ret;
|
| 2565 | unsigned long start_pfn = cc->zone->zone_start_pfn;
|
| 2566 | unsigned long end_pfn = zone_end_pfn(cc->zone);
|
| 2567 | unsigned long last_migrated_pfn;
|
| 2568 | const bool sync = cc->mode != MIGRATE_ASYNC;
|
| 2569 | bool update_cached;
|
| 2570 | unsigned int nr_succeeded = 0, nr_migratepages;
|
| 2571 | int order;
|
| 2572 |
|
| 2573 | |
| 2574 | * These counters track activities during zone compaction. Initialize
|
| 2575 | * them before compacting a new zone.
|
| 2576 |
|
| 2577 | cc->total_migrate_scanned = 0;
|
| 2578 | cc->total_free_scanned = 0;
|
| 2579 | cc->nr_migratepages = 0;
|
| 2580 | cc->nr_freepages = 0;
|
| 2581 | for (order = 0; order < NR_PAGE_ORDERS; order++)
|
| 2582 | INIT_LIST_HEAD(&cc->freepages[order]);
|
| 2583 | INIT_LIST_HEAD(&cc->migratepages);
|
| 2584 |
|
| 2585 | cc->migratetype = gfp_migratetype(cc->gfp_mask);
|
| 2586 |
|
| 2587 | if (!is_via_compact_memory(cc->order)) {
|
| 2588 | ret = compaction_suit_allocation_order(cc->zone, cc->order,
|
| 2589 | cc->highest_zoneidx,
|
| 2590 | cc->alloc_flags,
|
| 2591 | cc->mode == MIGRATE_ASYNC,
|
| 2592 | !cc->direct_compaction);
|
| 2593 | if (ret != COMPACT_CONTINUE)
|
| 2594 | return ret;
|
| 2595 | }
|
| 2596 |
|
| 2597 | |
| 2598 | * Clear pageblock skip if there were failures recently and compaction
|
| 2599 | * is about to be retried after being deferred.
|
| 2600 |
|
| 2601 | if (compaction_restarting(cc->zone, cc->order))
|
| 2602 | __reset_isolation_suitable(cc->zone);
|
| 2603 |
|
| 2604 | |
| 2605 | * Setup to move all movable pages to the end of the zone. Used cached
|
| 2606 | * information on where the scanners should start (unless we explicitly
|
| 2607 | * want to compact the whole zone), but check that it is initialised
|
| 2608 | * by ensuring the values are within zone boundaries.
|
| 2609 |
|
| 2610 | cc->fast_start_pfn = 0;
|
| 2611 | if (cc->whole_zone) {
|
| 2612 | cc->migrate_pfn = start_pfn;
|
| 2613 | cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
|
| 2614 | } else {
|
| 2615 | cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
|
| 2616 | cc->free_pfn = cc->zone->compact_cached_free_pfn;
|
| 2617 | if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
|
| 2618 | cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
|
| 2619 | cc->zone->compact_cached_free_pfn = cc->free_pfn;
|
| 2620 | }
|
| 2621 | if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
|
| 2622 | cc->migrate_pfn = start_pfn;
|
| 2623 | cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
|
| 2624 | cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
|
| 2625 | }
|
| 2626 |
|
| 2627 | if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)
|
| 2628 | cc->whole_zone = true;
|
| 2629 | }
|
| 2630 |
|
| 2631 | last_migrated_pfn = 0;
|
| 2632 |
|
| 2633 | |
| 2634 | * Migrate has separate cached PFNs for ASYNC and SYNC* migration on
|
| 2635 | * the basis that some migrations will fail in ASYNC mode. However,
|
| 2636 | * if the cached PFNs match and pageblocks are skipped due to having
|
| 2637 | * no isolation candidates, then the sync state does not matter.
|
| 2638 | * Until a pageblock with isolation candidates is found, keep the
|
| 2639 | * cached PFNs in sync to avoid revisiting the same blocks.
|
| 2640 |
|
| 2641 | update_cached = !sync &&
|
| 2642 | cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];
|
| 2643 |
|
| 2644 | trace_mm_compaction_begin(cc, start_pfn, end_pfn, sync);
|
| 2645 |
|
| 2646 |
|
| 2647 | lru_add_drain();
|
| 2648 |
|
| 2649 | while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
|
| 2650 | int err;
|
| 2651 | unsigned long iteration_start_pfn = cc->migrate_pfn;
|
| 2652 |
|
| 2653 | |
| 2654 | * Avoid multiple rescans of the same pageblock which can
|
| 2655 | * happen if a page cannot be isolated (dirty/writeback in
|
| 2656 | * async mode) or if the migrated pages are being allocated
|
| 2657 | * before the pageblock is cleared. The first rescan will
|
| 2658 | * capture the entire pageblock for migration. If it fails,
|
| 2659 | * it'll be marked skip and scanning will proceed as normal.
|
| 2660 |
|
| 2661 | cc->finish_pageblock = false;
|
| 2662 | if (pageblock_start_pfn(last_migrated_pfn) ==
|
| 2663 | pageblock_start_pfn(iteration_start_pfn)) {
|
| 2664 | cc->finish_pageblock = true;
|
| 2665 | }
|
| 2666 |
|
| 2667 | rescan:
|
| 2668 | switch (isolate_migratepages(cc)) {
|
| 2669 | case ISOLATE_ABORT:
|
| 2670 | ret = COMPACT_CONTENDED;
|
| 2671 | putback_movable_pages(&cc->migratepages);
|
| 2672 | cc->nr_migratepages = 0;
|
| 2673 | goto out;
|
| 2674 | case ISOLATE_NONE:
|
| 2675 | if (update_cached) {
|
| 2676 | cc->zone->compact_cached_migrate_pfn[1] =
|
| 2677 | cc->zone->compact_cached_migrate_pfn[0];
|
| 2678 | }
|
| 2679 |
|
| 2680 | |
| 2681 | * We haven't isolated and migrated anything, but
|
| 2682 | * there might still be unflushed migrations from
|
| 2683 | * previous cc->order aligned block.
|
| 2684 |
|
| 2685 | goto check_drain;
|
| 2686 | case ISOLATE_SUCCESS:
|
| 2687 | update_cached = false;
|
| 2688 | last_migrated_pfn = max(cc->zone->zone_start_pfn,
|
| 2689 | pageblock_start_pfn(cc->migrate_pfn - 1));
|
| 2690 | }
|
| 2691 |
|
| 2692 | |
| 2693 | * Record the number of pages to migrate since the
|
| 2694 | * compaction_alloc/free() will update cc->nr_migratepages
|
| 2695 | * properly.
|
| 2696 |
|
| 2697 | nr_migratepages = cc->nr_migratepages;
|
| 2698 | err = migrate_pages(&cc->migratepages, compaction_alloc,
|
| 2699 | compaction_free, (unsigned long)cc, cc->mode,
|
| 2700 | MR_COMPACTION, &nr_succeeded);
|
| 2701 |
|
| 2702 | trace_mm_compaction_migratepages(nr_migratepages, nr_succeeded);
|
| 2703 |
|
| 2704 |
|
| 2705 | cc->nr_migratepages = 0;
|
| 2706 | if (err) {
|
| 2707 | putback_movable_pages(&cc->migratepages);
|
| 2708 | |
| 2709 | * migrate_pages() may return -ENOMEM when scanners meet
|
| 2710 | * and we want compact_finished() to detect it
|
| 2711 |
|
| 2712 | if (err == -ENOMEM && !compact_scanners_met(cc)) {
|
| 2713 | ret = COMPACT_CONTENDED;
|
| 2714 | goto out;
|
| 2715 | }
|
| 2716 | |
| 2717 | * If an ASYNC or SYNC_LIGHT fails to migrate a page
|
| 2718 | * within the pageblock_order-aligned block and
|
| 2719 | * fast_find_migrateblock may be used then scan the
|
| 2720 | * remainder of the pageblock. This will mark the
|
| 2721 | * pageblock "skip" to avoid rescanning in the near
|
| 2722 | * future. This will isolate more pages than necessary
|
| 2723 | * for the request but avoid loops due to
|
| 2724 | * fast_find_migrateblock revisiting blocks that were
|
| 2725 | * recently partially scanned.
|
| 2726 |
|
| 2727 | if (!pageblock_aligned(cc->migrate_pfn) &&
|
| 2728 | !cc->ignore_skip_hint && !cc->finish_pageblock &&
|
| 2729 | (cc->mode < MIGRATE_SYNC)) {
|
| 2730 | cc->finish_pageblock = true;
|
| 2731 |
|
| 2732 | |
| 2733 | * Draining pcplists does not help THP if
|
| 2734 | * any page failed to migrate. Even after
|
| 2735 | * drain, the pageblock will not be free.
|
| 2736 |
|
| 2737 | if (cc->order == COMPACTION_HPAGE_ORDER)
|
| 2738 | last_migrated_pfn = 0;
|
| 2739 |
|
| 2740 | goto rescan;
|
| 2741 | }
|
| 2742 | }
|
| 2743 |
|
| 2744 |
|
| 2745 | if (capc && capc->page) {
|
| 2746 | ret = COMPACT_SUCCESS;
|
| 2747 | break;
|
| 2748 | }
|
| 2749 |
|
| 2750 | check_drain:
|
| 2751 | |
| 2752 | * Has the migration scanner moved away from the previous
|
| 2753 | * cc->order aligned block where we migrated from? If yes,
|
| 2754 | * flush the pages that were freed, so that they can merge and
|
| 2755 | * compact_finished() can detect immediately if allocation
|
| 2756 | * would succeed.
|
| 2757 |
|
| 2758 | if (cc->order > 0 && last_migrated_pfn) {
|
| 2759 | unsigned long current_block_start =
|
| 2760 | block_start_pfn(cc->migrate_pfn, cc->order);
|
| 2761 |
|
| 2762 | if (last_migrated_pfn < current_block_start) {
|
| 2763 | lru_add_drain_cpu_zone(cc->zone);
|
| 2764 |
|
| 2765 | last_migrated_pfn = 0;
|
| 2766 | }
|
| 2767 | }
|
| 2768 | }
|
| 2769 |
|
| 2770 | out:
|
| 2771 | |
| 2772 | * Release free pages and update where the free scanner should restart,
|
| 2773 | * so we don't leave any returned pages behind in the next attempt.
|
| 2774 |
|
| 2775 | if (cc->nr_freepages > 0) {
|
| 2776 | unsigned long free_pfn = release_free_list(cc->freepages);
|
| 2777 |
|
| 2778 | cc->nr_freepages = 0;
|
| 2779 | VM_BUG_ON(free_pfn == 0);
|
| 2780 |
|
| 2781 | free_pfn = pageblock_start_pfn(free_pfn);
|
| 2782 | |
| 2783 | * Only go back, not forward. The cached pfn might have been
|
| 2784 | * already reset to zone end in compact_finished()
|
| 2785 |
|
| 2786 | if (free_pfn > cc->zone->compact_cached_free_pfn)
|
| 2787 | cc->zone->compact_cached_free_pfn = free_pfn;
|
| 2788 | }
|
| 2789 |
|
| 2790 | count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
|
| 2791 | count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
|
| 2792 |
|
| 2793 | trace_mm_compaction_end(cc, start_pfn, end_pfn, sync, ret);
|
| 2794 |
|
| 2795 | VM_BUG_ON(!list_empty(&cc->migratepages));
|
| 2796 |
|
| 2797 | return ret;
|
| 2798 | }
|
| 2799 |
|
| 2800 | static enum compact_result compact_zone_order(struct zone *zone, int order,
|
| 2801 | gfp_t gfp_mask, enum compact_priority prio,
|
| 2802 | unsigned int alloc_flags, int highest_zoneidx,
|
| 2803 | struct capture_control *capc)
|
| 2804 | {
|
| 2805 | struct compact_control cc = {
|
| 2806 | .order = order,
|
| 2807 | .search_order = order,
|
| 2808 | .gfp_mask = gfp_mask,
|
| 2809 | .zone = zone,
|
| 2810 | .mode = (prio == COMPACT_PRIO_ASYNC) ?
|
| 2811 | MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
|
| 2812 | .alloc_flags = alloc_flags,
|
| 2813 | .highest_zoneidx = highest_zoneidx,
|
| 2814 | .direct_compaction = true,
|
| 2815 | .whole_zone = (prio == MIN_COMPACT_PRIORITY),
|
| 2816 | .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
|
| 2817 | .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
|
| 2818 | };
|
| 2819 |
|
| 2820 | return compact_zone(&cc, capc);
|
| 2821 | }
|
| 2822 |
|
| 2823 | |
| 2824 | * try_to_compact_pages - Direct compact to satisfy a high-order allocation
|
| 2825 | * @gfp_mask: The GFP mask of the current allocation
|
| 2826 | * @order: The order to try to make available
|
| 2827 | * @alloc_flags: The allocation flags of the current allocation
|
| 2828 | * @ac: The context of current allocation
|
| 2829 | * @prio: Determines how hard direct compaction should try to succeed
|
| 2830 | * @capc: Free page capture bypassing the freelist
|
| 2831 | *
|
| 2832 | * This is the main entry point for direct page compaction.
|
| 2833 |
|
| 2834 | enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
|
| 2835 | unsigned int alloc_flags, const struct alloc_context *ac,
|
| 2836 | enum compact_priority prio, struct capture_control *capc)
|
| 2837 | {
|
| 2838 | struct zoneref *z;
|
| 2839 | struct zone *zone;
|
| 2840 | enum compact_result rc = COMPACT_SKIPPED;
|
| 2841 |
|
| 2842 | if (!gfp_compaction_allowed(gfp_mask))
|
| 2843 | return COMPACT_SKIPPED;
|
| 2844 |
|
| 2845 | trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
|
| 2846 |
|
| 2847 |
|
| 2848 | for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
|
| 2849 | ac->highest_zoneidx, ac->nodemask) {
|
| 2850 | enum compact_result status;
|
| 2851 |
|
| 2852 | if (cpusets_enabled() &&
|
| 2853 | (alloc_flags & ALLOC_CPUSET) &&
|
| 2854 | !__cpuset_zone_allowed(zone, gfp_mask))
|
| 2855 | continue;
|
| 2856 |
|
| 2857 | if (prio > MIN_COMPACT_PRIORITY
|
| 2858 | && compaction_deferred(zone, order)) {
|
| 2859 | rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
|
| 2860 | continue;
|
| 2861 | }
|
| 2862 |
|
| 2863 | WRITE_ONCE(capc->zone, zone);
|
| 2864 |
|
| 2865 | status = compact_zone_order(zone, order, gfp_mask, prio,
|
| 2866 | alloc_flags, ac->highest_zoneidx, capc);
|
| 2867 |
|
| 2868 | WRITE_ONCE(capc->zone, NULL);
|
| 2869 |
|
| 2870 |
|
| 2871 | if (READ_ONCE(capc->page))
|
| 2872 | status = COMPACT_SUCCESS;
|
| 2873 |
|
| 2874 | rc = max(status, rc);
|
| 2875 |
|
| 2876 |
|
| 2877 | if (status == COMPACT_SUCCESS) {
|
| 2878 | |
| 2879 | * We think the allocation will succeed in this zone,
|
| 2880 | * but it is not certain, hence the false. The caller
|
| 2881 | * will repeat this with true if allocation indeed
|
| 2882 | * succeeds in this zone.
|
| 2883 |
|
| 2884 | compaction_defer_reset(zone, order, false);
|
| 2885 |
|
| 2886 | break;
|
| 2887 | }
|
| 2888 |
|
| 2889 | if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
|
| 2890 | status == COMPACT_PARTIAL_SKIPPED))
|
| 2891 | |
| 2892 | * We think that allocation won't succeed in this zone
|
| 2893 | * so we defer compaction there. If it ends up
|
| 2894 | * succeeding after all, it will be reset.
|
| 2895 |
|
| 2896 | defer_compaction(zone, order);
|
| 2897 |
|
| 2898 | |
| 2899 | * We might have stopped compacting due to need_resched() in
|
| 2900 | * async compaction, or due to a fatal signal detected. In that
|
| 2901 | * case do not try further zones
|
| 2902 |
|
| 2903 | if ((prio == COMPACT_PRIO_ASYNC && need_resched())
|
| 2904 | || fatal_signal_pending(current))
|
| 2905 | break;
|
| 2906 | }
|
| 2907 |
|
| 2908 | return rc;
|
| 2909 | }
|
| 2910 |
|
| 2911 | |
| 2912 | * compact_node() - compact all zones within a node
|
| 2913 | * @pgdat: The node page data
|
| 2914 | * @proactive: Whether the compaction is proactive
|
| 2915 | *
|
| 2916 | * For proactive compaction, compact till each zone's fragmentation score
|
| 2917 | * reaches within proactive compaction thresholds (as determined by the
|
| 2918 | * proactiveness tunable), it is possible that the function returns before
|
| 2919 | * reaching score targets due to various back-off conditions, such as,
|
| 2920 | * contention on per-node or per-zone locks.
|
| 2921 |
|
| 2922 | static int compact_node(pg_data_t *pgdat, bool proactive)
|
| 2923 | {
|
| 2924 | int zoneid;
|
| 2925 | struct zone *zone;
|
| 2926 | struct compact_control cc = {
|
| 2927 | .order = -1,
|
| 2928 | .mode = proactive ? MIGRATE_SYNC_LIGHT : MIGRATE_SYNC,
|
| 2929 | .ignore_skip_hint = true,
|
| 2930 | .whole_zone = true,
|
| 2931 | .gfp_mask = GFP_KERNEL,
|
| 2932 | .proactive_compaction = proactive,
|
| 2933 | };
|
| 2934 |
|
| 2935 | for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
|
| 2936 | zone = &pgdat->node_zones[zoneid];
|
| 2937 | if (!populated_zone(zone))
|
| 2938 | continue;
|
| 2939 |
|
| 2940 | if (fatal_signal_pending(current))
|
| 2941 | return -EINTR;
|
| 2942 |
|
| 2943 | cc.zone = zone;
|
| 2944 |
|
| 2945 | compact_zone(&cc, NULL);
|
| 2946 |
|
| 2947 | if (proactive) {
|
| 2948 | count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
|
| 2949 | cc.total_migrate_scanned);
|
| 2950 | count_compact_events(KCOMPACTD_FREE_SCANNED,
|
| 2951 | cc.total_free_scanned);
|
| 2952 | }
|
| 2953 | }
|
| 2954 |
|
| 2955 | return 0;
|
| 2956 | }
|
| 2957 |
|
| 2958 |
|
| 2959 | static int compact_nodes(void)
|
| 2960 | {
|
| 2961 | int ret, nid;
|
| 2962 |
|
| 2963 |
|
| 2964 | lru_add_drain_all();
|
| 2965 |
|
| 2966 | for_each_online_node(nid) {
|
| 2967 | ret = compact_node(NODE_DATA(nid), false);
|
| 2968 | if (ret)
|
| 2969 | return ret;
|
| 2970 | }
|
| 2971 |
|
| 2972 | return 0;
|
| 2973 | }
|
| 2974 |
|
| 2975 | static int compaction_proactiveness_sysctl_handler(const struct ctl_table *table, int write,
|
| 2976 | void *buffer, size_t *length, loff_t *ppos)
|
| 2977 | {
|
| 2978 | int rc, nid;
|
| 2979 |
|
| 2980 | rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
|
| 2981 | if (rc)
|
| 2982 | return rc;
|
| 2983 |
|
| 2984 | if (write && sysctl_compaction_proactiveness) {
|
| 2985 | for_each_online_node(nid) {
|
| 2986 | pg_data_t *pgdat = NODE_DATA(nid);
|
| 2987 |
|
| 2988 | if (pgdat->proactive_compact_trigger)
|
| 2989 | continue;
|
| 2990 |
|
| 2991 | pgdat->proactive_compact_trigger = true;
|
| 2992 | trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, -1,
|
| 2993 | pgdat->nr_zones - 1);
|
| 2994 | wake_up_interruptible(&pgdat->kcompactd_wait);
|
| 2995 | }
|
| 2996 | }
|
| 2997 |
|
| 2998 | return 0;
|
| 2999 | }
|
| 3000 |
|
| 3001 | |
| 3002 | * This is the entry point for compacting all nodes via
|
| 3003 | * /proc/sys/vm/compact_memory
|
| 3004 |
|
| 3005 | static int sysctl_compaction_handler(const struct ctl_table *table, int write,
|
| 3006 | void *buffer, size_t *length, loff_t *ppos)
|
| 3007 | {
|
| 3008 | int ret;
|
| 3009 |
|
| 3010 | ret = proc_dointvec(table, write, buffer, length, ppos);
|
| 3011 | if (ret)
|
| 3012 | return ret;
|
| 3013 |
|
| 3014 | if (sysctl_compact_memory != 1)
|
| 3015 | return -EINVAL;
|
| 3016 |
|
| 3017 | if (write)
|
| 3018 | ret = compact_nodes();
|
| 3019 |
|
| 3020 | return ret;
|
| 3021 | }
|
| 3022 |
|
| 3023 | #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
|
| 3024 | static ssize_t compact_store(struct device *dev,
|
| 3025 | struct device_attribute *attr,
|
| 3026 | const char *buf, size_t count)
|
| 3027 | {
|
| 3028 | int nid = dev->id;
|
| 3029 |
|
| 3030 | if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
|
| 3031 |
|
| 3032 | lru_add_drain_all();
|
| 3033 |
|
| 3034 | compact_node(NODE_DATA(nid), false);
|
| 3035 | }
|
| 3036 |
|
| 3037 | return count;
|
| 3038 | }
|
| 3039 | static DEVICE_ATTR_WO(compact);
|
| 3040 |
|
| 3041 | int compaction_register_node(struct node *node)
|
| 3042 | {
|
| 3043 | return device_create_file(&node->dev, &dev_attr_compact);
|
| 3044 | }
|
| 3045 |
|
| 3046 | void compaction_unregister_node(struct node *node)
|
| 3047 | {
|
| 3048 | device_remove_file(&node->dev, &dev_attr_compact);
|
| 3049 | }
|
| 3050 | #endif
|
| 3051 |
|
| 3052 | static inline bool kcompactd_work_requested(pg_data_t *pgdat)
|
| 3053 | {
|
| 3054 | return pgdat->kcompactd_max_order > 0 || kthread_should_stop() ||
|
| 3055 | pgdat->proactive_compact_trigger;
|
| 3056 | }
|
| 3057 |
|
| 3058 | static bool kcompactd_node_suitable(pg_data_t *pgdat)
|
| 3059 | {
|
| 3060 | int zoneid;
|
| 3061 | struct zone *zone;
|
| 3062 | enum zone_type highest_zoneidx = pgdat->kcompactd_highest_zoneidx;
|
| 3063 | enum compact_result ret;
|
| 3064 | unsigned int alloc_flags = defrag_mode ?
|
| 3065 | ALLOC_WMARK_HIGH : ALLOC_WMARK_MIN;
|
| 3066 |
|
| 3067 | for (zoneid = 0; zoneid <= highest_zoneidx; zoneid++) {
|
| 3068 | zone = &pgdat->node_zones[zoneid];
|
| 3069 |
|
| 3070 | if (!populated_zone(zone))
|
| 3071 | continue;
|
| 3072 |
|
| 3073 | ret = compaction_suit_allocation_order(zone,
|
| 3074 | pgdat->kcompactd_max_order,
|
| 3075 | highest_zoneidx, alloc_flags,
|
| 3076 | false, true);
|
| 3077 | if (ret == COMPACT_CONTINUE)
|
| 3078 | return true;
|
| 3079 | }
|
| 3080 |
|
| 3081 | return false;
|
| 3082 | }
|
| 3083 |
|
| 3084 | static void kcompactd_do_work(pg_data_t *pgdat)
|
| 3085 | {
|
| 3086 | |
| 3087 | * With no special task, compact all zones so that a page of requested
|
| 3088 | * order is allocatable.
|
| 3089 |
|
| 3090 | int zoneid;
|
| 3091 | struct zone *zone;
|
| 3092 | struct compact_control cc = {
|
| 3093 | .order = pgdat->kcompactd_max_order,
|
| 3094 | .search_order = pgdat->kcompactd_max_order,
|
| 3095 | .highest_zoneidx = pgdat->kcompactd_highest_zoneidx,
|
| 3096 | .mode = MIGRATE_SYNC_LIGHT,
|
| 3097 | .ignore_skip_hint = false,
|
| 3098 | .gfp_mask = GFP_KERNEL,
|
| 3099 | .alloc_flags = defrag_mode ? ALLOC_WMARK_HIGH : ALLOC_WMARK_MIN,
|
| 3100 | };
|
| 3101 | enum compact_result ret;
|
| 3102 |
|
| 3103 | trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
|
| 3104 | cc.highest_zoneidx);
|
| 3105 | count_compact_event(KCOMPACTD_WAKE);
|
| 3106 |
|
| 3107 | for (zoneid = 0; zoneid <= cc.highest_zoneidx; zoneid++) {
|
| 3108 | int status;
|
| 3109 |
|
| 3110 | zone = &pgdat->node_zones[zoneid];
|
| 3111 | if (!populated_zone(zone))
|
| 3112 | continue;
|
| 3113 |
|
| 3114 | if (compaction_deferred(zone, cc.order))
|
| 3115 | continue;
|
| 3116 |
|
| 3117 | ret = compaction_suit_allocation_order(zone,
|
| 3118 | cc.order, zoneid, cc.alloc_flags,
|
| 3119 | false, true);
|
| 3120 | if (ret != COMPACT_CONTINUE)
|
| 3121 | continue;
|
| 3122 |
|
| 3123 | if (kthread_should_stop())
|
| 3124 | return;
|
| 3125 |
|
| 3126 | cc.zone = zone;
|
| 3127 | status = compact_zone(&cc, NULL);
|
| 3128 |
|
| 3129 | if (status == COMPACT_SUCCESS) {
|
| 3130 | compaction_defer_reset(zone, cc.order, false);
|
| 3131 | } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
|
| 3132 | |
| 3133 | * Buddy pages may become stranded on pcps that could
|
| 3134 | * otherwise coalesce on the zone's free area for
|
| 3135 | * order >= cc.order. This is ratelimited by the
|
| 3136 | * upcoming deferral.
|
| 3137 |
|
| 3138 | drain_all_pages(zone);
|
| 3139 |
|
| 3140 | |
| 3141 | * We use sync migration mode here, so we defer like
|
| 3142 | * sync direct compaction does.
|
| 3143 |
|
| 3144 | defer_compaction(zone, cc.order);
|
| 3145 | }
|
| 3146 |
|
| 3147 | count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
|
| 3148 | cc.total_migrate_scanned);
|
| 3149 | count_compact_events(KCOMPACTD_FREE_SCANNED,
|
| 3150 | cc.total_free_scanned);
|
| 3151 | }
|
| 3152 |
|
| 3153 | |
| 3154 | * Regardless of success, we are done until woken up next. But remember
|
| 3155 | * the requested order/highest_zoneidx in case it was higher/tighter
|
| 3156 | * than our current ones
|
| 3157 |
|
| 3158 | if (pgdat->kcompactd_max_order <= cc.order)
|
| 3159 | pgdat->kcompactd_max_order = 0;
|
| 3160 | if (pgdat->kcompactd_highest_zoneidx >= cc.highest_zoneidx)
|
| 3161 | pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
|
| 3162 | }
|
| 3163 |
|
| 3164 | void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)
|
| 3165 | {
|
| 3166 | if (!order)
|
| 3167 | return;
|
| 3168 |
|
| 3169 | if (pgdat->kcompactd_max_order < order)
|
| 3170 | pgdat->kcompactd_max_order = order;
|
| 3171 |
|
| 3172 | if (pgdat->kcompactd_highest_zoneidx > highest_zoneidx)
|
| 3173 | pgdat->kcompactd_highest_zoneidx = highest_zoneidx;
|
| 3174 |
|
| 3175 | |
| 3176 | * Pairs with implicit barrier in wait_event_freezable()
|
| 3177 | * such that wakeups are not missed.
|
| 3178 |
|
| 3179 | if (!wq_has_sleeper(&pgdat->kcompactd_wait))
|
| 3180 | return;
|
| 3181 |
|
| 3182 | if (!kcompactd_node_suitable(pgdat))
|
| 3183 | return;
|
| 3184 |
|
| 3185 | trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
|
| 3186 | highest_zoneidx);
|
| 3187 | wake_up_interruptible(&pgdat->kcompactd_wait);
|
| 3188 | }
|
| 3189 |
|
| 3190 | |
| 3191 | * The background compaction daemon, started as a kernel thread
|
| 3192 | * from the init process.
|
| 3193 |
|
| 3194 | static int kcompactd(void *p)
|
| 3195 | {
|
| 3196 | pg_data_t *pgdat = (pg_data_t *)p;
|
| 3197 | long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC);
|
| 3198 | long timeout = default_timeout;
|
| 3199 |
|
| 3200 | current->flags |= PF_KCOMPACTD;
|
| 3201 | set_freezable();
|
| 3202 |
|
| 3203 | pgdat->kcompactd_max_order = 0;
|
| 3204 | pgdat->kcompactd_highest_zoneidx = pgdat->nr_zones - 1;
|
| 3205 |
|
| 3206 | while (!kthread_should_stop()) {
|
| 3207 | unsigned long pflags;
|
| 3208 |
|
| 3209 | |
| 3210 | * Avoid the unnecessary wakeup for proactive compaction
|
| 3211 | * when it is disabled.
|
| 3212 |
|
| 3213 | if (!sysctl_compaction_proactiveness)
|
| 3214 | timeout = MAX_SCHEDULE_TIMEOUT;
|
| 3215 | trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
|
| 3216 | if (wait_event_freezable_timeout(pgdat->kcompactd_wait,
|
| 3217 | kcompactd_work_requested(pgdat), timeout) &&
|
| 3218 | !pgdat->proactive_compact_trigger) {
|
| 3219 |
|
| 3220 | psi_memstall_enter(&pflags);
|
| 3221 | kcompactd_do_work(pgdat);
|
| 3222 | psi_memstall_leave(&pflags);
|
| 3223 | |
| 3224 | * Reset the timeout value. The defer timeout from
|
| 3225 | * proactive compaction is lost here but that is fine
|
| 3226 | * as the condition of the zone changing substantionally
|
| 3227 | * then carrying on with the previous defer interval is
|
| 3228 | * not useful.
|
| 3229 |
|
| 3230 | timeout = default_timeout;
|
| 3231 | continue;
|
| 3232 | }
|
| 3233 |
|
| 3234 | |
| 3235 | * Start the proactive work with default timeout. Based
|
| 3236 | * on the fragmentation score, this timeout is updated.
|
| 3237 |
|
| 3238 | timeout = default_timeout;
|
| 3239 | if (should_proactive_compact_node(pgdat)) {
|
| 3240 | unsigned int prev_score, score;
|
| 3241 |
|
| 3242 | prev_score = fragmentation_score_node(pgdat);
|
| 3243 | compact_node(pgdat, true);
|
| 3244 | score = fragmentation_score_node(pgdat);
|
| 3245 | |
| 3246 | * Defer proactive compaction if the fragmentation
|
| 3247 | * score did not go down i.e. no progress made.
|
| 3248 |
|
| 3249 | if (unlikely(score >= prev_score))
|
| 3250 | timeout =
|
| 3251 | default_timeout << COMPACT_MAX_DEFER_SHIFT;
|
| 3252 | }
|
| 3253 | if (unlikely(pgdat->proactive_compact_trigger))
|
| 3254 | pgdat->proactive_compact_trigger = false;
|
| 3255 | }
|
| 3256 |
|
| 3257 | current->flags &= ~PF_KCOMPACTD;
|
| 3258 |
|
| 3259 | return 0;
|
| 3260 | }
|
| 3261 |
|
| 3262 | |
| 3263 | * This kcompactd start function will be called by init and node-hot-add.
|
| 3264 | * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
|
| 3265 |
|
| 3266 | void __meminit kcompactd_run(int nid)
|
| 3267 | {
|
| 3268 | pg_data_t *pgdat = NODE_DATA(nid);
|
| 3269 |
|
| 3270 | if (pgdat->kcompactd)
|
| 3271 | return;
|
| 3272 |
|
| 3273 | pgdat->kcompactd = kthread_create_on_node(kcompactd, pgdat, nid, "kcompactd%d", nid);
|
| 3274 | if (IS_ERR(pgdat->kcompactd)) {
|
| 3275 | pr_err("Failed to start kcompactd on node %d\n", nid);
|
| 3276 | pgdat->kcompactd = NULL;
|
| 3277 | } else {
|
| 3278 | wake_up_process(pgdat->kcompactd);
|
| 3279 | }
|
| 3280 | }
|
| 3281 |
|
| 3282 | |
| 3283 | * Called by memory hotplug when all memory in a node is offlined. Caller must
|
| 3284 | * be holding mem_hotplug_begin/done().
|
| 3285 |
|
| 3286 | void __meminit kcompactd_stop(int nid)
|
| 3287 | {
|
| 3288 | struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
|
| 3289 |
|
| 3290 | if (kcompactd) {
|
| 3291 | kthread_stop(kcompactd);
|
| 3292 | NODE_DATA(nid)->kcompactd = NULL;
|
| 3293 | }
|
| 3294 | }
|
| 3295 |
|
| 3296 | static int proc_dointvec_minmax_warn_RT_change(const struct ctl_table *table,
|
| 3297 | int write, void *buffer, size_t *lenp, loff_t *ppos)
|
| 3298 | {
|
| 3299 | int ret, old;
|
| 3300 |
|
| 3301 | if (!IS_ENABLED(CONFIG_PREEMPT_RT) || !write)
|
| 3302 | return proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
| 3303 |
|
| 3304 | old = *(int *)table->data;
|
| 3305 | ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
|
| 3306 | if (ret)
|
| 3307 | return ret;
|
| 3308 | if (old != *(int *)table->data)
|
| 3309 | pr_warn_once("sysctl attribute %s changed by %s[%d]\n",
|
| 3310 | table->procname, current->comm,
|
| 3311 | task_pid_nr(current));
|
| 3312 | return ret;
|
| 3313 | }
|
| 3314 |
|
| 3315 | static const struct ctl_table vm_compaction[] = {
|
| 3316 | {
|
| 3317 | .procname = "compact_memory",
|
| 3318 | .data = &sysctl_compact_memory,
|
| 3319 | .maxlen = sizeof(int),
|
| 3320 | .mode = 0200,
|
| 3321 | .proc_handler = sysctl_compaction_handler,
|
| 3322 | },
|
| 3323 | {
|
| 3324 | .procname = "compaction_proactiveness",
|
| 3325 | .data = &sysctl_compaction_proactiveness,
|
| 3326 | .maxlen = sizeof(sysctl_compaction_proactiveness),
|
| 3327 | .mode = 0644,
|
| 3328 | .proc_handler = compaction_proactiveness_sysctl_handler,
|
| 3329 | .extra1 = SYSCTL_ZERO,
|
| 3330 | .extra2 = SYSCTL_ONE_HUNDRED,
|
| 3331 | },
|
| 3332 | {
|
| 3333 | .procname = "extfrag_threshold",
|
| 3334 | .data = &sysctl_extfrag_threshold,
|
| 3335 | .maxlen = sizeof(int),
|
| 3336 | .mode = 0644,
|
| 3337 | .proc_handler = proc_dointvec_minmax,
|
| 3338 | .extra1 = SYSCTL_ZERO,
|
| 3339 | .extra2 = SYSCTL_ONE_THOUSAND,
|
| 3340 | },
|
| 3341 | {
|
| 3342 | .procname = "compact_unevictable_allowed",
|
| 3343 | .data = &sysctl_compact_unevictable_allowed,
|
| 3344 | .maxlen = sizeof(int),
|
| 3345 | .mode = 0644,
|
| 3346 | .proc_handler = proc_dointvec_minmax_warn_RT_change,
|
| 3347 | .extra1 = SYSCTL_ZERO,
|
| 3348 | .extra2 = SYSCTL_ONE,
|
| 3349 | },
|
| 3350 | };
|
| 3351 |
|
| 3352 | static int __init kcompactd_init(void)
|
| 3353 | {
|
| 3354 | int nid;
|
| 3355 |
|
| 3356 | for_each_node_state(nid, N_MEMORY)
|
| 3357 | kcompactd_run(nid);
|
| 3358 | register_sysctl_init("vm", vm_compaction);
|
| 3359 | return 0;
|
| 3360 | }
|
| 3361 | subsys_initcall(kcompactd_init)
|
| 3362 |
|
| 3363 | #endif
|