| 1 | |
| 2 | |
| 3 | * Generic hugetlb support.
|
| 4 | * (C) Nadia Yvette Chambers, April 2004
|
| 5 |
|
| 6 | #include <linux/list.h>
|
| 7 | #include <linux/init.h>
|
| 8 | #include <linux/mm.h>
|
| 9 | #include <linux/seq_file.h>
|
| 10 | #include <linux/highmem.h>
|
| 11 | #include <linux/mmu_notifier.h>
|
| 12 | #include <linux/nodemask.h>
|
| 13 | #include <linux/pagemap.h>
|
| 14 | #include <linux/mempolicy.h>
|
| 15 | #include <linux/compiler.h>
|
| 16 | #include <linux/cpumask.h>
|
| 17 | #include <linux/cpuset.h>
|
| 18 | #include <linux/mutex.h>
|
| 19 | #include <linux/memblock.h>
|
| 20 | #include <linux/minmax.h>
|
| 21 | #include <linux/slab.h>
|
| 22 | #include <linux/sched/mm.h>
|
| 23 | #include <linux/mmdebug.h>
|
| 24 | #include <linux/sched/signal.h>
|
| 25 | #include <linux/rmap.h>
|
| 26 | #include <linux/string_choices.h>
|
| 27 | #include <linux/string_helpers.h>
|
| 28 | #include <linux/swap.h>
|
| 29 | #include <linux/leafops.h>
|
| 30 | #include <linux/jhash.h>
|
| 31 | #include <linux/numa.h>
|
| 32 | #include <linux/llist.h>
|
| 33 | #include <linux/cma.h>
|
| 34 | #include <linux/migrate.h>
|
| 35 | #include <linux/nospec.h>
|
| 36 | #include <linux/delayacct.h>
|
| 37 | #include <linux/memory.h>
|
| 38 | #include <linux/mm_inline.h>
|
| 39 | #include <linux/padata.h>
|
| 40 | #include <linux/pgalloc.h>
|
| 41 |
|
| 42 | #include <asm/page.h>
|
| 43 | #include <asm/tlb.h>
|
| 44 | #include <asm/setup.h>
|
| 45 |
|
| 46 | #include <linux/io.h>
|
| 47 | #include <linux/node.h>
|
| 48 | #include <linux/page_owner.h>
|
| 49 | #include "internal.h"
|
| 50 | #include "page_alloc.h"
|
| 51 | #include "hugetlb_vmemmap.h"
|
| 52 | #include "hugetlb_cma.h"
|
| 53 | #include "hugetlb_internal.h"
|
| 54 | #include "mm_init.h"
|
| 55 | #include <linux/page-isolation.h>
|
| 56 |
|
| 57 | int hugetlb_max_hstate __read_mostly;
|
| 58 | unsigned int default_hstate_idx;
|
| 59 | struct hstate hstates[HUGE_MAX_HSTATE];
|
| 60 |
|
| 61 | __initdata nodemask_t hugetlb_bootmem_nodes;
|
| 62 | __initdata struct list_head huge_boot_pages[MAX_NUMNODES];
|
| 63 |
|
| 64 | |
| 65 | * Due to ordering constraints across the init code for various
|
| 66 | * architectures, hugetlb hstate cmdline parameters can't simply
|
| 67 | * be early_param. early_param might call the setup function
|
| 68 | * before valid hugetlb page sizes are determined, leading to
|
| 69 | * incorrect rejection of valid hugepagesz= options.
|
| 70 | *
|
| 71 | * So, record the parameters early and consume them whenever the
|
| 72 | * init code is ready for them, by calling hugetlb_parse_params().
|
| 73 |
|
| 74 |
|
| 75 |
|
| 76 | #define HUGE_MAX_CMDLINE_ARGS (2 * HUGE_MAX_HSTATE + 1)
|
| 77 | struct hugetlb_cmdline {
|
| 78 | char *val;
|
| 79 | int (*setup)(char *val);
|
| 80 | };
|
| 81 |
|
| 82 |
|
| 83 | static struct hstate * __initdata parsed_hstate;
|
| 84 | static unsigned long __initdata default_hstate_max_huge_pages;
|
| 85 | static bool __initdata parsed_valid_hugepagesz = true;
|
| 86 | static bool __initdata parsed_default_hugepagesz;
|
| 87 | static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
|
| 88 | static unsigned long hugepage_allocation_threads __initdata;
|
| 89 |
|
| 90 | static char hstate_cmdline_buf[COMMAND_LINE_SIZE] __initdata;
|
| 91 | static int hstate_cmdline_index __initdata;
|
| 92 | static struct hugetlb_cmdline hugetlb_params[HUGE_MAX_CMDLINE_ARGS] __initdata;
|
| 93 | static int hugetlb_param_index __initdata;
|
| 94 | static __init int hugetlb_add_param(char *s, int (*setup)(char *val));
|
| 95 | static __init void hugetlb_parse_params(void);
|
| 96 |
|
| 97 | #define hugetlb_early_param(str, func) \
|
| 98 | static __init int func##args(char *s) \
|
| 99 | { \
|
| 100 | return hugetlb_add_param(s, func); \
|
| 101 | } \
|
| 102 | early_param(str, func##args)
|
| 103 |
|
| 104 | |
| 105 | * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
|
| 106 | * free_huge_pages, and surplus_huge_pages.
|
| 107 |
|
| 108 | __cacheline_aligned_in_smp DEFINE_SPINLOCK(hugetlb_lock);
|
| 109 |
|
| 110 | |
| 111 | * Serializes faults on the same logical page. This is used to
|
| 112 | * prevent spurious OOMs when the hugepage pool is fully utilized.
|
| 113 |
|
| 114 | static int num_fault_mutexes __ro_after_init;
|
| 115 | struct mutex *hugetlb_fault_mutex_table __ro_after_init;
|
| 116 |
|
| 117 |
|
| 118 | static int hugetlb_acct_memory(struct hstate *h, long delta);
|
| 119 | static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
|
| 120 | static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
|
| 121 | static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
|
| 122 | static int __huge_pmd_unshare(struct mmu_gather *tlb,
|
| 123 | struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
|
| 124 | bool check_locks);
|
| 125 | static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
|
| 126 | unsigned long start, unsigned long end, bool take_locks);
|
| 127 | static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
|
| 128 |
|
| 129 | static inline bool subpool_is_free(struct hugepage_subpool *spool)
|
| 130 | {
|
| 131 | if (spool->count)
|
| 132 | return false;
|
| 133 | if (spool->max_hpages != -1)
|
| 134 | return spool->used_hpages == 0;
|
| 135 | if (spool->min_hpages != -1)
|
| 136 | return spool->rsv_hpages == spool->min_hpages;
|
| 137 |
|
| 138 | return true;
|
| 139 | }
|
| 140 |
|
| 141 | static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
|
| 142 | unsigned long irq_flags)
|
| 143 | {
|
| 144 | bool free_subpool = subpool_is_free(spool);
|
| 145 |
|
| 146 | |
| 147 | * remain, give up any reservations based on minimum size and
|
| 148 | * free the subpool
|
| 149 | spin_unlock_irqrestore(&spool->lock, irq_flags);
|
| 150 |
|
| 151 | if (free_subpool) {
|
| 152 | if (spool->min_hpages != -1)
|
| 153 | hugetlb_acct_memory(spool->hstate,
|
| 154 | -spool->min_hpages);
|
| 155 | kfree(spool);
|
| 156 | }
|
| 157 | }
|
| 158 |
|
| 159 | struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
|
| 160 | long min_hpages)
|
| 161 | {
|
| 162 | struct hugepage_subpool *spool;
|
| 163 |
|
| 164 | spool = kzalloc_obj(*spool);
|
| 165 | if (!spool)
|
| 166 | return NULL;
|
| 167 |
|
| 168 | spin_lock_init(&spool->lock);
|
| 169 | spool->count = 1;
|
| 170 | spool->max_hpages = max_hpages;
|
| 171 | spool->hstate = h;
|
| 172 | spool->min_hpages = min_hpages;
|
| 173 |
|
| 174 | if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
|
| 175 | kfree(spool);
|
| 176 | return NULL;
|
| 177 | }
|
| 178 | spool->rsv_hpages = min_hpages;
|
| 179 |
|
| 180 | return spool;
|
| 181 | }
|
| 182 |
|
| 183 | void hugepage_put_subpool(struct hugepage_subpool *spool)
|
| 184 | {
|
| 185 | unsigned long flags;
|
| 186 |
|
| 187 | if (!spool)
|
| 188 | return;
|
| 189 |
|
| 190 | spin_lock_irqsave(&spool->lock, flags);
|
| 191 | BUG_ON(!spool->count);
|
| 192 | spool->count--;
|
| 193 | unlock_or_release_subpool(spool, flags);
|
| 194 | }
|
| 195 |
|
| 196 | |
| 197 | * Subpool accounting for allocating and reserving pages.
|
| 198 | * Return -ENOMEM if there are not enough resources to satisfy the
|
| 199 | * request. Otherwise, return the number of pages by which the
|
| 200 | * global pools must be adjusted (upward). The returned value may
|
| 201 | * only be different than the passed value (delta) in the case where
|
| 202 | * a subpool minimum size must be maintained.
|
| 203 |
|
| 204 | static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
|
| 205 | long delta)
|
| 206 | {
|
| 207 | long ret = delta;
|
| 208 |
|
| 209 | if (!spool)
|
| 210 | return ret;
|
| 211 |
|
| 212 | spin_lock_irq(&spool->lock);
|
| 213 |
|
| 214 | if (spool->max_hpages != -1) {
|
| 215 | if ((spool->used_hpages + delta) <= spool->max_hpages)
|
| 216 | spool->used_hpages += delta;
|
| 217 | else {
|
| 218 | ret = -ENOMEM;
|
| 219 | goto unlock_ret;
|
| 220 | }
|
| 221 | }
|
| 222 |
|
| 223 |
|
| 224 | if (spool->min_hpages != -1 && spool->rsv_hpages) {
|
| 225 | if (delta > spool->rsv_hpages) {
|
| 226 | |
| 227 | * Asking for more reserves than those already taken on
|
| 228 | * behalf of subpool. Return difference.
|
| 229 |
|
| 230 | ret = delta - spool->rsv_hpages;
|
| 231 | spool->rsv_hpages = 0;
|
| 232 | } else {
|
| 233 | ret = 0;
|
| 234 | spool->rsv_hpages -= delta;
|
| 235 | }
|
| 236 | }
|
| 237 |
|
| 238 | unlock_ret:
|
| 239 | spin_unlock_irq(&spool->lock);
|
| 240 | return ret;
|
| 241 | }
|
| 242 |
|
| 243 | |
| 244 | * Subpool accounting for freeing and unreserving pages.
|
| 245 | * Return the number of global page reservations that must be dropped.
|
| 246 | * The return value may only be different than the passed value (delta)
|
| 247 | * in the case where a subpool minimum size must be maintained.
|
| 248 |
|
| 249 | static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
|
| 250 | long delta)
|
| 251 | {
|
| 252 | long ret = delta;
|
| 253 | unsigned long flags;
|
| 254 |
|
| 255 | if (!spool)
|
| 256 | return delta;
|
| 257 |
|
| 258 | spin_lock_irqsave(&spool->lock, flags);
|
| 259 |
|
| 260 | if (spool->max_hpages != -1)
|
| 261 | spool->used_hpages -= delta;
|
| 262 |
|
| 263 |
|
| 264 | if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
|
| 265 | if (spool->rsv_hpages + delta <= spool->min_hpages)
|
| 266 | ret = 0;
|
| 267 | else
|
| 268 | ret = spool->rsv_hpages + delta - spool->min_hpages;
|
| 269 |
|
| 270 | spool->rsv_hpages += delta;
|
| 271 | if (spool->rsv_hpages > spool->min_hpages)
|
| 272 | spool->rsv_hpages = spool->min_hpages;
|
| 273 | }
|
| 274 |
|
| 275 | |
| 276 | * If hugetlbfs_put_super couldn't free spool due to an outstanding
|
| 277 | * quota reference, free it now.
|
| 278 |
|
| 279 | unlock_or_release_subpool(spool, flags);
|
| 280 |
|
| 281 | return ret;
|
| 282 | }
|
| 283 |
|
| 284 | static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
|
| 285 | {
|
| 286 | return subpool_inode(file_inode(vma->vm_file));
|
| 287 | }
|
| 288 |
|
| 289 | |
| 290 | * hugetlb vma_lock helper routines
|
| 291 |
|
| 292 | void hugetlb_vma_lock_read(struct vm_area_struct *vma)
|
| 293 | {
|
| 294 | if (__vma_shareable_lock(vma)) {
|
| 295 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 296 |
|
| 297 | down_read(&vma_lock->rw_sema);
|
| 298 | } else if (__vma_private_lock(vma)) {
|
| 299 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 300 |
|
| 301 | down_read(&resv_map->rw_sema);
|
| 302 | }
|
| 303 | }
|
| 304 |
|
| 305 | void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
|
| 306 | {
|
| 307 | if (__vma_shareable_lock(vma)) {
|
| 308 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 309 |
|
| 310 | up_read(&vma_lock->rw_sema);
|
| 311 | } else if (__vma_private_lock(vma)) {
|
| 312 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 313 |
|
| 314 | up_read(&resv_map->rw_sema);
|
| 315 | }
|
| 316 | }
|
| 317 |
|
| 318 | void hugetlb_vma_lock_write(struct vm_area_struct *vma)
|
| 319 | {
|
| 320 | if (__vma_shareable_lock(vma)) {
|
| 321 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 322 |
|
| 323 | down_write(&vma_lock->rw_sema);
|
| 324 | } else if (__vma_private_lock(vma)) {
|
| 325 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 326 |
|
| 327 | down_write(&resv_map->rw_sema);
|
| 328 | }
|
| 329 | }
|
| 330 |
|
| 331 | void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
|
| 332 | {
|
| 333 | if (__vma_shareable_lock(vma)) {
|
| 334 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 335 |
|
| 336 | up_write(&vma_lock->rw_sema);
|
| 337 | } else if (__vma_private_lock(vma)) {
|
| 338 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 339 |
|
| 340 | up_write(&resv_map->rw_sema);
|
| 341 | }
|
| 342 | }
|
| 343 |
|
| 344 | int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
|
| 345 | {
|
| 346 |
|
| 347 | if (__vma_shareable_lock(vma)) {
|
| 348 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 349 |
|
| 350 | return down_write_trylock(&vma_lock->rw_sema);
|
| 351 | } else if (__vma_private_lock(vma)) {
|
| 352 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 353 |
|
| 354 | return down_write_trylock(&resv_map->rw_sema);
|
| 355 | }
|
| 356 |
|
| 357 | return 1;
|
| 358 | }
|
| 359 |
|
| 360 | void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
|
| 361 | {
|
| 362 | if (__vma_shareable_lock(vma)) {
|
| 363 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 364 |
|
| 365 | lockdep_assert_held(&vma_lock->rw_sema);
|
| 366 | } else if (__vma_private_lock(vma)) {
|
| 367 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 368 |
|
| 369 | lockdep_assert_held(&resv_map->rw_sema);
|
| 370 | }
|
| 371 | }
|
| 372 |
|
| 373 | void hugetlb_vma_lock_release(struct kref *kref)
|
| 374 | {
|
| 375 | struct hugetlb_vma_lock *vma_lock = container_of(kref,
|
| 376 | struct hugetlb_vma_lock, refs);
|
| 377 |
|
| 378 | kfree(vma_lock);
|
| 379 | }
|
| 380 |
|
| 381 | static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
|
| 382 | {
|
| 383 | struct vm_area_struct *vma = vma_lock->vma;
|
| 384 |
|
| 385 | |
| 386 | * vma_lock structure may or not be released as a result of put,
|
| 387 | * it certainly will no longer be attached to vma so clear pointer.
|
| 388 | * Semaphore synchronizes access to vma_lock->vma field.
|
| 389 |
|
| 390 | vma_lock->vma = NULL;
|
| 391 | vma->vm_private_data = NULL;
|
| 392 | up_write(&vma_lock->rw_sema);
|
| 393 | kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
|
| 394 | }
|
| 395 |
|
| 396 | static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
|
| 397 | {
|
| 398 | if (__vma_shareable_lock(vma)) {
|
| 399 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 400 |
|
| 401 | __hugetlb_vma_unlock_write_put(vma_lock);
|
| 402 | } else if (__vma_private_lock(vma)) {
|
| 403 | struct resv_map *resv_map = vma_resv_map(vma);
|
| 404 |
|
| 405 |
|
| 406 | up_write(&resv_map->rw_sema);
|
| 407 | }
|
| 408 | }
|
| 409 |
|
| 410 | static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
|
| 411 | {
|
| 412 | |
| 413 | * Only present in sharable vmas.
|
| 414 |
|
| 415 | if (!vma || !__vma_shareable_lock(vma))
|
| 416 | return;
|
| 417 |
|
| 418 | if (vma->vm_private_data) {
|
| 419 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 420 |
|
| 421 | down_write(&vma_lock->rw_sema);
|
| 422 | __hugetlb_vma_unlock_write_put(vma_lock);
|
| 423 | }
|
| 424 | }
|
| 425 |
|
| 426 | static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
|
| 427 | {
|
| 428 | struct hugetlb_vma_lock *vma_lock;
|
| 429 |
|
| 430 |
|
| 431 | if (!vma || !(vma->vm_flags & VM_MAYSHARE))
|
| 432 | return;
|
| 433 |
|
| 434 |
|
| 435 | if (vma->vm_private_data)
|
| 436 | return;
|
| 437 |
|
| 438 | vma_lock = kmalloc_obj(*vma_lock);
|
| 439 | if (!vma_lock) {
|
| 440 | |
| 441 | * If we can not allocate structure, then vma can not
|
| 442 | * participate in pmd sharing. This is only a possible
|
| 443 | * performance enhancement and memory saving issue.
|
| 444 | * However, the lock is also used to synchronize page
|
| 445 | * faults with truncation. If the lock is not present,
|
| 446 | * unlikely races could leave pages in a file past i_size
|
| 447 | * until the file is removed. Warn in the unlikely case of
|
| 448 | * allocation failure.
|
| 449 |
|
| 450 | pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
|
| 451 | return;
|
| 452 | }
|
| 453 |
|
| 454 | kref_init(&vma_lock->refs);
|
| 455 | init_rwsem(&vma_lock->rw_sema);
|
| 456 | vma_lock->vma = vma;
|
| 457 | vma->vm_private_data = vma_lock;
|
| 458 | }
|
| 459 |
|
| 460 | |
| 461 | * it for use.
|
| 462 |
|
| 463 | static struct file_region *
|
| 464 | get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
|
| 465 | {
|
| 466 | struct file_region *nrg;
|
| 467 |
|
| 468 | VM_BUG_ON(resv->region_cache_count <= 0);
|
| 469 |
|
| 470 | resv->region_cache_count--;
|
| 471 | nrg = list_first_entry(&resv->region_cache, struct file_region, link);
|
| 472 | list_del(&nrg->link);
|
| 473 |
|
| 474 | nrg->from = from;
|
| 475 | nrg->to = to;
|
| 476 |
|
| 477 | return nrg;
|
| 478 | }
|
| 479 |
|
| 480 | static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
|
| 481 | struct file_region *rg)
|
| 482 | {
|
| 483 | #ifdef CONFIG_CGROUP_HUGETLB
|
| 484 | nrg->reservation_counter = rg->reservation_counter;
|
| 485 | nrg->css = rg->css;
|
| 486 | if (rg->css)
|
| 487 | css_get(rg->css);
|
| 488 | #endif
|
| 489 | }
|
| 490 |
|
| 491 |
|
| 492 | static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
|
| 493 | struct hstate *h,
|
| 494 | struct resv_map *resv,
|
| 495 | struct file_region *nrg)
|
| 496 | {
|
| 497 | #ifdef CONFIG_CGROUP_HUGETLB
|
| 498 | if (h_cg) {
|
| 499 | nrg->reservation_counter =
|
| 500 | &h_cg->rsvd_hugepage[hstate_index(h)];
|
| 501 | nrg->css = &h_cg->css;
|
| 502 | |
| 503 | * The caller will hold exactly one h_cg->css reference for the
|
| 504 | * whole contiguous reservation region. But this area might be
|
| 505 | * scattered when there are already some file_regions reside in
|
| 506 | * it. As a result, many file_regions may share only one css
|
| 507 | * reference. In order to ensure that one file_region must hold
|
| 508 | * exactly one h_cg->css reference, we should do css_get for
|
| 509 | * each file_region and leave the reference held by caller
|
| 510 | * untouched.
|
| 511 |
|
| 512 | css_get(&h_cg->css);
|
| 513 | if (!resv->pages_per_hpage)
|
| 514 | resv->pages_per_hpage = pages_per_huge_page(h);
|
| 515 | |
| 516 | * a resv_map.
|
| 517 |
|
| 518 | VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
|
| 519 | } else {
|
| 520 | nrg->reservation_counter = NULL;
|
| 521 | nrg->css = NULL;
|
| 522 | }
|
| 523 | #endif
|
| 524 | }
|
| 525 |
|
| 526 | static void put_uncharge_info(struct file_region *rg)
|
| 527 | {
|
| 528 | #ifdef CONFIG_CGROUP_HUGETLB
|
| 529 | if (rg->css)
|
| 530 | css_put(rg->css);
|
| 531 | #endif
|
| 532 | }
|
| 533 |
|
| 534 | static bool has_same_uncharge_info(struct file_region *rg,
|
| 535 | struct file_region *org)
|
| 536 | {
|
| 537 | #ifdef CONFIG_CGROUP_HUGETLB
|
| 538 | return rg->reservation_counter == org->reservation_counter &&
|
| 539 | rg->css == org->css;
|
| 540 |
|
| 541 | #else
|
| 542 | return true;
|
| 543 | #endif
|
| 544 | }
|
| 545 |
|
| 546 | static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
|
| 547 | {
|
| 548 | struct file_region *nrg, *prg;
|
| 549 |
|
| 550 | prg = list_prev_entry(rg, link);
|
| 551 | if (&prg->link != &resv->regions && prg->to == rg->from &&
|
| 552 | has_same_uncharge_info(prg, rg)) {
|
| 553 | prg->to = rg->to;
|
| 554 |
|
| 555 | list_del(&rg->link);
|
| 556 | put_uncharge_info(rg);
|
| 557 | kfree(rg);
|
| 558 |
|
| 559 | rg = prg;
|
| 560 | }
|
| 561 |
|
| 562 | nrg = list_next_entry(rg, link);
|
| 563 | if (&nrg->link != &resv->regions && nrg->from == rg->to &&
|
| 564 | has_same_uncharge_info(nrg, rg)) {
|
| 565 | nrg->from = rg->from;
|
| 566 |
|
| 567 | list_del(&rg->link);
|
| 568 | put_uncharge_info(rg);
|
| 569 | kfree(rg);
|
| 570 | }
|
| 571 | }
|
| 572 |
|
| 573 | static inline long
|
| 574 | hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
|
| 575 | long to, struct hstate *h, struct hugetlb_cgroup *cg,
|
| 576 | long *regions_needed)
|
| 577 | {
|
| 578 | struct file_region *nrg;
|
| 579 |
|
| 580 | if (!regions_needed) {
|
| 581 | nrg = get_file_region_entry_from_cache(map, from, to);
|
| 582 | record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
|
| 583 | list_add(&nrg->link, rg);
|
| 584 | coalesce_file_region(map, nrg);
|
| 585 | } else {
|
| 586 | *regions_needed += 1;
|
| 587 | }
|
| 588 |
|
| 589 | return to - from;
|
| 590 | }
|
| 591 |
|
| 592 | |
| 593 | * Must be called with resv->lock held.
|
| 594 | *
|
| 595 | * Calling this with regions_needed != NULL will count the number of pages
|
| 596 | * to be added but will not modify the linked list. And regions_needed will
|
| 597 | * indicate the number of file_regions needed in the cache to carry out to add
|
| 598 | * the regions for this range.
|
| 599 |
|
| 600 | static long add_reservation_in_range(struct resv_map *resv, long f, long t,
|
| 601 | struct hugetlb_cgroup *h_cg,
|
| 602 | struct hstate *h, long *regions_needed)
|
| 603 | {
|
| 604 | long add = 0;
|
| 605 | struct list_head *head = &resv->regions;
|
| 606 | long last_accounted_offset = f;
|
| 607 | struct file_region *iter, *trg = NULL;
|
| 608 | struct list_head *rg = NULL;
|
| 609 |
|
| 610 | if (regions_needed)
|
| 611 | *regions_needed = 0;
|
| 612 |
|
| 613 | |
| 614 | * [last_accounted_offset, iter->from), at every iteration, with some
|
| 615 | * bounds checking.
|
| 616 |
|
| 617 | list_for_each_entry_safe(iter, trg, head, link) {
|
| 618 |
|
| 619 | if (iter->from < f) {
|
| 620 | |
| 621 | * then we need to update last_accounted_offset.
|
| 622 |
|
| 623 | if (iter->to > last_accounted_offset)
|
| 624 | last_accounted_offset = iter->to;
|
| 625 | continue;
|
| 626 | }
|
| 627 |
|
| 628 | |
| 629 | * finished.
|
| 630 |
|
| 631 | if (iter->from >= t) {
|
| 632 | rg = iter->link.prev;
|
| 633 | break;
|
| 634 | }
|
| 635 |
|
| 636 | |
| 637 | * update last_accounted_offset.
|
| 638 |
|
| 639 | if (iter->from > last_accounted_offset)
|
| 640 | add += hugetlb_resv_map_add(resv, iter->link.prev,
|
| 641 | last_accounted_offset,
|
| 642 | iter->from, h, h_cg,
|
| 643 | regions_needed);
|
| 644 |
|
| 645 | last_accounted_offset = iter->to;
|
| 646 | }
|
| 647 |
|
| 648 | |
| 649 | * last_accounted_offset.
|
| 650 |
|
| 651 | if (!rg)
|
| 652 | rg = head->prev;
|
| 653 | if (last_accounted_offset < t)
|
| 654 | add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
|
| 655 | t, h, h_cg, regions_needed);
|
| 656 |
|
| 657 | return add;
|
| 658 | }
|
| 659 |
|
| 660 | |
| 661 |
|
| 662 | static int allocate_file_region_entries(struct resv_map *resv,
|
| 663 | int regions_needed)
|
| 664 | __must_hold(&resv->lock)
|
| 665 | {
|
| 666 | LIST_HEAD(allocated_regions);
|
| 667 | int to_allocate = 0, i = 0;
|
| 668 | struct file_region *trg = NULL, *rg = NULL;
|
| 669 |
|
| 670 | VM_BUG_ON(regions_needed < 0);
|
| 671 |
|
| 672 | |
| 673 | * Check for sufficient descriptors in the cache to accommodate
|
| 674 | * the number of in progress add operations plus regions_needed.
|
| 675 | *
|
| 676 | * This is a while loop because when we drop the lock, some other call
|
| 677 | * to region_add or region_del may have consumed some region_entries,
|
| 678 | * so we keep looping here until we finally have enough entries for
|
| 679 | * (adds_in_progress + regions_needed).
|
| 680 |
|
| 681 | while (resv->region_cache_count <
|
| 682 | (resv->adds_in_progress + regions_needed)) {
|
| 683 | to_allocate = resv->adds_in_progress + regions_needed -
|
| 684 | resv->region_cache_count;
|
| 685 |
|
| 686 | |
| 687 | * for all the existing adds_in_progress. We should only be
|
| 688 | * needing to allocate for regions_needed.
|
| 689 |
|
| 690 | VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
|
| 691 |
|
| 692 | spin_unlock(&resv->lock);
|
| 693 | for (i = 0; i < to_allocate; i++) {
|
| 694 | trg = kmalloc_obj(*trg);
|
| 695 | if (!trg)
|
| 696 | goto out_of_memory;
|
| 697 | list_add(&trg->link, &allocated_regions);
|
| 698 | }
|
| 699 |
|
| 700 | spin_lock(&resv->lock);
|
| 701 |
|
| 702 | list_splice_init(&allocated_regions, &resv->region_cache);
|
| 703 | resv->region_cache_count += to_allocate;
|
| 704 | }
|
| 705 |
|
| 706 | return 0;
|
| 707 |
|
| 708 | out_of_memory:
|
| 709 | list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
|
| 710 | list_del(&rg->link);
|
| 711 | kfree(rg);
|
| 712 | }
|
| 713 | return -ENOMEM;
|
| 714 | }
|
| 715 |
|
| 716 | |
| 717 | * Add the huge page range represented by [f, t) to the reserve
|
| 718 | * map. Regions will be taken from the cache to fill in this range.
|
| 719 | * Sufficient regions should exist in the cache due to the previous
|
| 720 | * call to region_chg with the same range, but in some cases the cache will not
|
| 721 | * have sufficient entries due to races with other code doing region_add or
|
| 722 | * region_del. The extra needed entries will be allocated.
|
| 723 | *
|
| 724 | * regions_needed is the out value provided by a previous call to region_chg.
|
| 725 | *
|
| 726 | * Return the number of new huge pages added to the map. This number is greater
|
| 727 | * than or equal to zero. If file_region entries needed to be allocated for
|
| 728 | * this operation and we were not able to allocate, it returns -ENOMEM.
|
| 729 | * region_add of regions of length 1 never allocate file_regions and cannot
|
| 730 | * fail; region_chg will always allocate at least 1 entry and a region_add for
|
| 731 | * 1 page will only require at most 1 entry.
|
| 732 |
|
| 733 | static long region_add(struct resv_map *resv, long f, long t,
|
| 734 | long in_regions_needed, struct hstate *h,
|
| 735 | struct hugetlb_cgroup *h_cg)
|
| 736 | {
|
| 737 | long add = 0, actual_regions_needed = 0;
|
| 738 |
|
| 739 | spin_lock(&resv->lock);
|
| 740 | retry:
|
| 741 |
|
| 742 |
|
| 743 | add_reservation_in_range(resv, f, t, NULL, NULL,
|
| 744 | &actual_regions_needed);
|
| 745 |
|
| 746 | |
| 747 | * Check for sufficient descriptors in the cache to accommodate
|
| 748 | * this add operation. Note that actual_regions_needed may be greater
|
| 749 | * than in_regions_needed, as the resv_map may have been modified since
|
| 750 | * the region_chg call. In this case, we need to make sure that we
|
| 751 | * allocate extra entries, such that we have enough for all the
|
| 752 | * existing adds_in_progress, plus the excess needed for this
|
| 753 | * operation.
|
| 754 |
|
| 755 | if (actual_regions_needed > in_regions_needed &&
|
| 756 | resv->region_cache_count <
|
| 757 | resv->adds_in_progress +
|
| 758 | (actual_regions_needed - in_regions_needed)) {
|
| 759 | |
| 760 | * allocate file_region entries.
|
| 761 |
|
| 762 | VM_BUG_ON(t - f <= 1);
|
| 763 |
|
| 764 | if (allocate_file_region_entries(
|
| 765 | resv, actual_regions_needed - in_regions_needed)) {
|
| 766 | return -ENOMEM;
|
| 767 | }
|
| 768 |
|
| 769 | goto retry;
|
| 770 | }
|
| 771 |
|
| 772 | add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
|
| 773 |
|
| 774 | resv->adds_in_progress -= in_regions_needed;
|
| 775 |
|
| 776 | spin_unlock(&resv->lock);
|
| 777 | return add;
|
| 778 | }
|
| 779 |
|
| 780 | |
| 781 | * Examine the existing reserve map and determine how many
|
| 782 | * huge pages in the specified range [f, t) are NOT currently
|
| 783 | * represented. This routine is called before a subsequent
|
| 784 | * call to region_add that will actually modify the reserve
|
| 785 | * map to add the specified range [f, t). region_chg does
|
| 786 | * not change the number of huge pages represented by the
|
| 787 | * map. A number of new file_region structures is added to the cache as a
|
| 788 | * placeholder, for the subsequent region_add call to use. At least 1
|
| 789 | * file_region structure is added.
|
| 790 | *
|
| 791 | * out_regions_needed is the number of regions added to the
|
| 792 | * resv->adds_in_progress. This value needs to be provided to a follow up call
|
| 793 | * to region_add or region_abort for proper accounting.
|
| 794 | *
|
| 795 | * Returns the number of huge pages that need to be added to the existing
|
| 796 | * reservation map for the range [f, t). This number is greater or equal to
|
| 797 | * zero. -ENOMEM is returned if a new file_region structure or cache entry
|
| 798 | * is needed and can not be allocated.
|
| 799 |
|
| 800 | static long region_chg(struct resv_map *resv, long f, long t,
|
| 801 | long *out_regions_needed)
|
| 802 | {
|
| 803 | long chg = 0;
|
| 804 |
|
| 805 | spin_lock(&resv->lock);
|
| 806 |
|
| 807 |
|
| 808 | chg = add_reservation_in_range(resv, f, t, NULL, NULL,
|
| 809 | out_regions_needed);
|
| 810 |
|
| 811 | if (*out_regions_needed == 0)
|
| 812 | *out_regions_needed = 1;
|
| 813 |
|
| 814 | if (allocate_file_region_entries(resv, *out_regions_needed))
|
| 815 | return -ENOMEM;
|
| 816 |
|
| 817 | resv->adds_in_progress += *out_regions_needed;
|
| 818 |
|
| 819 | spin_unlock(&resv->lock);
|
| 820 | return chg;
|
| 821 | }
|
| 822 |
|
| 823 | |
| 824 | * Abort the in progress add operation. The adds_in_progress field
|
| 825 | * of the resv_map keeps track of the operations in progress between
|
| 826 | * calls to region_chg and region_add. Operations are sometimes
|
| 827 | * aborted after the call to region_chg. In such cases, region_abort
|
| 828 | * is called to decrement the adds_in_progress counter. regions_needed
|
| 829 | * is the value returned by the region_chg call, it is used to decrement
|
| 830 | * the adds_in_progress counter.
|
| 831 | *
|
| 832 | * NOTE: The range arguments [f, t) are not needed or used in this
|
| 833 | * routine. They are kept to make reading the calling code easier as
|
| 834 | * arguments will match the associated region_chg call.
|
| 835 |
|
| 836 | static void region_abort(struct resv_map *resv, long f, long t,
|
| 837 | long regions_needed)
|
| 838 | {
|
| 839 | spin_lock(&resv->lock);
|
| 840 | VM_BUG_ON(!resv->region_cache_count);
|
| 841 | resv->adds_in_progress -= regions_needed;
|
| 842 | spin_unlock(&resv->lock);
|
| 843 | }
|
| 844 |
|
| 845 | |
| 846 | * Delete the specified range [f, t) from the reserve map. If the
|
| 847 | * t parameter is LONG_MAX, this indicates that ALL regions after f
|
| 848 | * should be deleted. Locate the regions which intersect [f, t)
|
| 849 | * and either trim, delete or split the existing regions.
|
| 850 | *
|
| 851 | * Returns the number of huge pages deleted from the reserve map.
|
| 852 | * In the normal case, the return value is zero or more. In the
|
| 853 | * case where a region must be split, a new region descriptor must
|
| 854 | * be allocated. If the allocation fails, -ENOMEM will be returned.
|
| 855 | * NOTE: If the parameter t == LONG_MAX, then we will never split
|
| 856 | * a region and possibly return -ENOMEM. Callers specifying
|
| 857 | * t == LONG_MAX do not need to check for -ENOMEM error.
|
| 858 |
|
| 859 | static long region_del(struct resv_map *resv, long f, long t)
|
| 860 | {
|
| 861 | struct list_head *head = &resv->regions;
|
| 862 | struct file_region *rg, *trg;
|
| 863 | struct file_region *nrg = NULL;
|
| 864 | long del = 0;
|
| 865 |
|
| 866 | retry:
|
| 867 | spin_lock(&resv->lock);
|
| 868 | list_for_each_entry_safe(rg, trg, head, link) {
|
| 869 | |
| 870 | * Skip regions before the range to be deleted. file_region
|
| 871 | * ranges are normally of the form [from, to). However, there
|
| 872 | * may be a "placeholder" entry in the map which is of the form
|
| 873 | * (from, to) with from == to. Check for placeholder entries
|
| 874 | * at the beginning of the range to be deleted.
|
| 875 |
|
| 876 | if (rg->to <= f && (rg->to != rg->from || rg->to != f))
|
| 877 | continue;
|
| 878 |
|
| 879 | if (rg->from >= t)
|
| 880 | break;
|
| 881 |
|
| 882 | if (f > rg->from && t < rg->to) {
|
| 883 | |
| 884 | * Check for an entry in the cache before dropping
|
| 885 | * lock and attempting allocation.
|
| 886 |
|
| 887 | if (!nrg &&
|
| 888 | resv->region_cache_count > resv->adds_in_progress) {
|
| 889 | nrg = list_first_entry(&resv->region_cache,
|
| 890 | struct file_region,
|
| 891 | link);
|
| 892 | list_del(&nrg->link);
|
| 893 | resv->region_cache_count--;
|
| 894 | }
|
| 895 |
|
| 896 | if (!nrg) {
|
| 897 | spin_unlock(&resv->lock);
|
| 898 | nrg = kmalloc_obj(*nrg);
|
| 899 | if (!nrg)
|
| 900 | return -ENOMEM;
|
| 901 | goto retry;
|
| 902 | }
|
| 903 |
|
| 904 | del += t - f;
|
| 905 | hugetlb_cgroup_uncharge_file_region(
|
| 906 | resv, rg, t - f, false);
|
| 907 |
|
| 908 |
|
| 909 | nrg->from = t;
|
| 910 | nrg->to = rg->to;
|
| 911 |
|
| 912 | copy_hugetlb_cgroup_uncharge_info(nrg, rg);
|
| 913 |
|
| 914 | INIT_LIST_HEAD(&nrg->link);
|
| 915 |
|
| 916 |
|
| 917 | rg->to = f;
|
| 918 |
|
| 919 | list_add(&nrg->link, &rg->link);
|
| 920 | nrg = NULL;
|
| 921 | break;
|
| 922 | }
|
| 923 |
|
| 924 | if (f <= rg->from && t >= rg->to) {
|
| 925 | del += rg->to - rg->from;
|
| 926 | hugetlb_cgroup_uncharge_file_region(resv, rg,
|
| 927 | rg->to - rg->from, true);
|
| 928 | list_del(&rg->link);
|
| 929 | kfree(rg);
|
| 930 | continue;
|
| 931 | }
|
| 932 |
|
| 933 | if (f <= rg->from) {
|
| 934 | hugetlb_cgroup_uncharge_file_region(resv, rg,
|
| 935 | t - rg->from, false);
|
| 936 |
|
| 937 | del += t - rg->from;
|
| 938 | rg->from = t;
|
| 939 | } else {
|
| 940 | hugetlb_cgroup_uncharge_file_region(resv, rg,
|
| 941 | rg->to - f, false);
|
| 942 |
|
| 943 | del += rg->to - f;
|
| 944 | rg->to = f;
|
| 945 | }
|
| 946 | }
|
| 947 |
|
| 948 | spin_unlock(&resv->lock);
|
| 949 | kfree(nrg);
|
| 950 | return del;
|
| 951 | }
|
| 952 |
|
| 953 | |
| 954 | * A rare out of memory error was encountered which prevented removal of
|
| 955 | * the reserve map region for a page. The huge page itself was free'ed
|
| 956 | * and removed from the page cache. This routine will adjust the subpool
|
| 957 | * usage count, and the global reserve count if needed. By incrementing
|
| 958 | * these counts, the reserve map entry which could not be deleted will
|
| 959 | * appear as a "reserved" entry instead of simply dangling with incorrect
|
| 960 | * counts.
|
| 961 |
|
| 962 | void hugetlb_fix_reserve_counts(struct inode *inode)
|
| 963 | {
|
| 964 | struct hugepage_subpool *spool = subpool_inode(inode);
|
| 965 | long rsv_adjust;
|
| 966 | bool reserved = false;
|
| 967 |
|
| 968 | rsv_adjust = hugepage_subpool_get_pages(spool, 1);
|
| 969 | if (rsv_adjust > 0) {
|
| 970 | struct hstate *h = hstate_inode(inode);
|
| 971 |
|
| 972 | if (!hugetlb_acct_memory(h, 1))
|
| 973 | reserved = true;
|
| 974 | } else if (!rsv_adjust) {
|
| 975 | reserved = true;
|
| 976 | }
|
| 977 |
|
| 978 | if (!reserved)
|
| 979 | pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
|
| 980 | }
|
| 981 |
|
| 982 | |
| 983 | * Count and return the number of huge pages in the reserve map
|
| 984 | * that intersect with the range [f, t).
|
| 985 |
|
| 986 | static long region_count(struct resv_map *resv, long f, long t)
|
| 987 | {
|
| 988 | struct list_head *head = &resv->regions;
|
| 989 | struct file_region *rg;
|
| 990 | long chg = 0;
|
| 991 |
|
| 992 | spin_lock(&resv->lock);
|
| 993 |
|
| 994 | list_for_each_entry(rg, head, link) {
|
| 995 | long seg_from;
|
| 996 | long seg_to;
|
| 997 |
|
| 998 | if (rg->to <= f)
|
| 999 | continue;
|
| 1000 | if (rg->from >= t)
|
| 1001 | break;
|
| 1002 |
|
| 1003 | seg_from = max(rg->from, f);
|
| 1004 | seg_to = min(rg->to, t);
|
| 1005 |
|
| 1006 | chg += seg_to - seg_from;
|
| 1007 | }
|
| 1008 | spin_unlock(&resv->lock);
|
| 1009 |
|
| 1010 | return chg;
|
| 1011 | }
|
| 1012 |
|
| 1013 | |
| 1014 | * Convert the address within this vma to the page offset within
|
| 1015 | * the mapping, huge page units here.
|
| 1016 |
|
| 1017 | static pgoff_t vma_hugecache_offset(struct hstate *h,
|
| 1018 | struct vm_area_struct *vma, unsigned long address)
|
| 1019 | {
|
| 1020 | return linear_page_index(vma, address) >> huge_page_order(h);
|
| 1021 | }
|
| 1022 |
|
| 1023 | |
| 1024 | * Flags for MAP_PRIVATE reservations. These are stored in the bottom
|
| 1025 | * bits of the reservation map pointer, which are always clear due to
|
| 1026 | * alignment.
|
| 1027 |
|
| 1028 | #define HPAGE_RESV_OWNER (1UL << 0)
|
| 1029 | #define HPAGE_RESV_UNMAPPED (1UL << 1)
|
| 1030 | #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
|
| 1031 |
|
| 1032 | |
| 1033 | * These helpers are used to track how many pages are reserved for
|
| 1034 | * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
|
| 1035 | * is guaranteed to have their future faults succeed.
|
| 1036 | *
|
| 1037 | * With the exception of hugetlb_dup_vma_private() which is called at fork(),
|
| 1038 | * the reserve counters are updated with the hugetlb_lock held. It is safe
|
| 1039 | * to reset the VMA at fork() time as it is not in use yet and there is no
|
| 1040 | * chance of the global counters getting corrupted as a result of the values.
|
| 1041 | *
|
| 1042 | * The private mapping reservation is represented in a subtly different
|
| 1043 | * manner to a shared mapping. A shared mapping has a region map associated
|
| 1044 | * with the underlying file, this region map represents the backing file
|
| 1045 | * pages which have ever had a reservation assigned which this persists even
|
| 1046 | * after the page is instantiated. A private mapping has a region map
|
| 1047 | * associated with the original mmap which is attached to all VMAs which
|
| 1048 | * reference it, this region map represents those offsets which have consumed
|
| 1049 | * reservation ie. where pages have been instantiated.
|
| 1050 |
|
| 1051 | static unsigned long get_vma_private_data(struct vm_area_struct *vma)
|
| 1052 | {
|
| 1053 | return (unsigned long)vma->vm_private_data;
|
| 1054 | }
|
| 1055 |
|
| 1056 | static void set_vma_private_data(struct vm_area_struct *vma,
|
| 1057 | unsigned long value)
|
| 1058 | {
|
| 1059 | vma->vm_private_data = (void *)value;
|
| 1060 | }
|
| 1061 |
|
| 1062 | static void
|
| 1063 | resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
|
| 1064 | struct hugetlb_cgroup *h_cg,
|
| 1065 | struct hstate *h)
|
| 1066 | {
|
| 1067 | #ifdef CONFIG_CGROUP_HUGETLB
|
| 1068 | if (!h_cg || !h) {
|
| 1069 | resv_map->reservation_counter = NULL;
|
| 1070 | resv_map->pages_per_hpage = 0;
|
| 1071 | resv_map->css = NULL;
|
| 1072 | } else {
|
| 1073 | resv_map->reservation_counter =
|
| 1074 | &h_cg->rsvd_hugepage[hstate_index(h)];
|
| 1075 | resv_map->pages_per_hpage = pages_per_huge_page(h);
|
| 1076 | resv_map->css = &h_cg->css;
|
| 1077 | }
|
| 1078 | #endif
|
| 1079 | }
|
| 1080 |
|
| 1081 | struct resv_map *resv_map_alloc(void)
|
| 1082 | {
|
| 1083 | struct resv_map *resv_map = kmalloc_obj(*resv_map);
|
| 1084 | struct file_region *rg = kmalloc_obj(*rg);
|
| 1085 |
|
| 1086 | if (!resv_map || !rg) {
|
| 1087 | kfree(resv_map);
|
| 1088 | kfree(rg);
|
| 1089 | return NULL;
|
| 1090 | }
|
| 1091 |
|
| 1092 | kref_init(&resv_map->refs);
|
| 1093 | spin_lock_init(&resv_map->lock);
|
| 1094 | INIT_LIST_HEAD(&resv_map->regions);
|
| 1095 | init_rwsem(&resv_map->rw_sema);
|
| 1096 |
|
| 1097 | resv_map->adds_in_progress = 0;
|
| 1098 | |
| 1099 | * Initialize these to 0. On shared mappings, 0's here indicate these
|
| 1100 | * fields don't do cgroup accounting. On private mappings, these will be
|
| 1101 | * re-initialized to the proper values, to indicate that hugetlb cgroup
|
| 1102 | * reservations are to be un-charged from here.
|
| 1103 |
|
| 1104 | resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
|
| 1105 |
|
| 1106 | INIT_LIST_HEAD(&resv_map->region_cache);
|
| 1107 | list_add(&rg->link, &resv_map->region_cache);
|
| 1108 | resv_map->region_cache_count = 1;
|
| 1109 |
|
| 1110 | return resv_map;
|
| 1111 | }
|
| 1112 |
|
| 1113 | void resv_map_release(struct kref *ref)
|
| 1114 | {
|
| 1115 | struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
|
| 1116 | struct list_head *head = &resv_map->region_cache;
|
| 1117 | struct file_region *rg, *trg;
|
| 1118 |
|
| 1119 |
|
| 1120 | region_del(resv_map, 0, LONG_MAX);
|
| 1121 |
|
| 1122 |
|
| 1123 | list_for_each_entry_safe(rg, trg, head, link) {
|
| 1124 | list_del(&rg->link);
|
| 1125 | kfree(rg);
|
| 1126 | }
|
| 1127 |
|
| 1128 | VM_BUG_ON(resv_map->adds_in_progress);
|
| 1129 |
|
| 1130 | kfree(resv_map);
|
| 1131 | }
|
| 1132 |
|
| 1133 | static inline struct resv_map *inode_resv_map(struct inode *inode)
|
| 1134 | {
|
| 1135 | return HUGETLBFS_I(inode)->resv_map;
|
| 1136 | }
|
| 1137 |
|
| 1138 | static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
|
| 1139 | {
|
| 1140 | VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
|
| 1141 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 1142 | struct address_space *mapping = vma->vm_file->f_mapping;
|
| 1143 | struct inode *inode = mapping->host;
|
| 1144 |
|
| 1145 | return inode_resv_map(inode);
|
| 1146 |
|
| 1147 | } else {
|
| 1148 | return (struct resv_map *)(get_vma_private_data(vma) &
|
| 1149 | ~HPAGE_RESV_MASK);
|
| 1150 | }
|
| 1151 | }
|
| 1152 |
|
| 1153 | static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
|
| 1154 | {
|
| 1155 | VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
|
| 1156 | VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
|
| 1157 |
|
| 1158 | set_vma_private_data(vma, (unsigned long)map);
|
| 1159 | }
|
| 1160 |
|
| 1161 | static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
|
| 1162 | {
|
| 1163 | VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
|
| 1164 | VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
|
| 1165 |
|
| 1166 | set_vma_private_data(vma, get_vma_private_data(vma) | flags);
|
| 1167 | }
|
| 1168 |
|
| 1169 | static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
|
| 1170 | {
|
| 1171 | VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
|
| 1172 |
|
| 1173 | return (get_vma_private_data(vma) & flag) != 0;
|
| 1174 | }
|
| 1175 |
|
| 1176 | bool __vma_private_lock(struct vm_area_struct *vma)
|
| 1177 | {
|
| 1178 | return !(vma->vm_flags & VM_MAYSHARE) &&
|
| 1179 | get_vma_private_data(vma) & ~HPAGE_RESV_MASK &&
|
| 1180 | is_vma_resv_set(vma, HPAGE_RESV_OWNER);
|
| 1181 | }
|
| 1182 |
|
| 1183 | void hugetlb_dup_vma_private(struct vm_area_struct *vma)
|
| 1184 | {
|
| 1185 | VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
|
| 1186 | |
| 1187 | * Clear vm_private_data
|
| 1188 | * - For shared mappings this is a per-vma semaphore that may be
|
| 1189 | * allocated in a subsequent call to hugetlb_vm_op_open.
|
| 1190 | * Before clearing, make sure pointer is not associated with vma
|
| 1191 | * as this will leak the structure. This is the case when called
|
| 1192 | * via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already
|
| 1193 | * been called to allocate a new structure.
|
| 1194 | * - For MAP_PRIVATE mappings, this is the reserve map which does
|
| 1195 | * not apply to children. Faults generated by the children are
|
| 1196 | * not guaranteed to succeed, even if read-only.
|
| 1197 |
|
| 1198 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 1199 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 1200 |
|
| 1201 | if (vma_lock && vma_lock->vma != vma)
|
| 1202 | vma->vm_private_data = NULL;
|
| 1203 | } else {
|
| 1204 | vma->vm_private_data = NULL;
|
| 1205 | }
|
| 1206 | }
|
| 1207 |
|
| 1208 | |
| 1209 | * Reset and decrement one ref on hugepage private reservation.
|
| 1210 | * Called with mm->mmap_lock writer semaphore held.
|
| 1211 | * This function should be only used by mremap and operate on
|
| 1212 | * same sized vma. It should never come here with last ref on the
|
| 1213 | * reservation.
|
| 1214 |
|
| 1215 | void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
|
| 1216 | {
|
| 1217 | |
| 1218 | * Clear the old hugetlb private page reservation.
|
| 1219 | * It has already been transferred to new_vma.
|
| 1220 | *
|
| 1221 | * During a mremap() operation of a hugetlb vma we call move_vma()
|
| 1222 | * which copies vma into new_vma and unmaps vma. After the copy
|
| 1223 | * operation both new_vma and vma share a reference to the resv_map
|
| 1224 | * struct, and at that point vma is about to be unmapped. We don't
|
| 1225 | * want to return the reservation to the pool at unmap of vma because
|
| 1226 | * the reservation still lives on in new_vma, so simply decrement the
|
| 1227 | * ref here and remove the resv_map reference from this vma.
|
| 1228 |
|
| 1229 | struct resv_map *reservations = vma_resv_map(vma);
|
| 1230 |
|
| 1231 | if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
|
| 1232 | resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
|
| 1233 | kref_put(&reservations->refs, resv_map_release);
|
| 1234 | }
|
| 1235 |
|
| 1236 | hugetlb_dup_vma_private(vma);
|
| 1237 | }
|
| 1238 |
|
| 1239 | static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
|
| 1240 | {
|
| 1241 | int nid = folio_nid(folio);
|
| 1242 |
|
| 1243 | lockdep_assert_held(&hugetlb_lock);
|
| 1244 | VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
|
| 1245 |
|
| 1246 | list_move(&folio->lru, &h->hugepage_freelists[nid]);
|
| 1247 | h->free_huge_pages++;
|
| 1248 | h->free_huge_pages_node[nid]++;
|
| 1249 | folio_set_hugetlb_freed(folio);
|
| 1250 | }
|
| 1251 |
|
| 1252 | static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h,
|
| 1253 | int nid)
|
| 1254 | {
|
| 1255 | struct folio *folio;
|
| 1256 | bool pin = !!(current->flags & PF_MEMALLOC_PIN);
|
| 1257 |
|
| 1258 | lockdep_assert_held(&hugetlb_lock);
|
| 1259 | list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) {
|
| 1260 | if (pin && !folio_is_longterm_pinnable(folio))
|
| 1261 | continue;
|
| 1262 |
|
| 1263 | if (folio_test_hwpoison(folio))
|
| 1264 | continue;
|
| 1265 |
|
| 1266 | if (is_migrate_isolate_page(&folio->page))
|
| 1267 | continue;
|
| 1268 |
|
| 1269 | list_move(&folio->lru, &h->hugepage_activelist);
|
| 1270 | folio_ref_unfreeze(folio, 1);
|
| 1271 | folio_clear_hugetlb_freed(folio);
|
| 1272 | h->free_huge_pages--;
|
| 1273 | h->free_huge_pages_node[nid]--;
|
| 1274 | return folio;
|
| 1275 | }
|
| 1276 |
|
| 1277 | return NULL;
|
| 1278 | }
|
| 1279 |
|
| 1280 | static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_mask,
|
| 1281 | int nid, nodemask_t *nmask)
|
| 1282 | {
|
| 1283 | unsigned int cpuset_mems_cookie;
|
| 1284 | struct zonelist *zonelist;
|
| 1285 | struct zone *zone;
|
| 1286 | struct zoneref *z;
|
| 1287 | int node = NUMA_NO_NODE;
|
| 1288 |
|
| 1289 |
|
| 1290 | if (nid == NUMA_NO_NODE)
|
| 1291 | nid = numa_node_id();
|
| 1292 |
|
| 1293 | zonelist = node_zonelist(nid, gfp_mask);
|
| 1294 |
|
| 1295 | retry_cpuset:
|
| 1296 | cpuset_mems_cookie = read_mems_allowed_begin();
|
| 1297 | for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
|
| 1298 | struct folio *folio;
|
| 1299 |
|
| 1300 | if (!cpuset_zone_allowed(zone, gfp_mask))
|
| 1301 | continue;
|
| 1302 | |
| 1303 | * no need to ask again on the same node. Pool is node rather than
|
| 1304 | * zone aware
|
| 1305 |
|
| 1306 | if (zone_to_nid(zone) == node)
|
| 1307 | continue;
|
| 1308 | node = zone_to_nid(zone);
|
| 1309 |
|
| 1310 | folio = dequeue_hugetlb_folio_node_exact(h, node);
|
| 1311 | if (folio)
|
| 1312 | return folio;
|
| 1313 | }
|
| 1314 | if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
|
| 1315 | goto retry_cpuset;
|
| 1316 |
|
| 1317 | return NULL;
|
| 1318 | }
|
| 1319 |
|
| 1320 | static unsigned long available_huge_pages(struct hstate *h)
|
| 1321 | {
|
| 1322 | return h->free_huge_pages - h->resv_huge_pages;
|
| 1323 | }
|
| 1324 |
|
| 1325 | static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
|
| 1326 | struct mempolicy_interpreted *mpoli)
|
| 1327 | {
|
| 1328 | nodemask_t *nodemask = mpoli->nodemask;
|
| 1329 | struct folio *folio = NULL;
|
| 1330 |
|
| 1331 | if (mpoli->mode == MPOL_PREFERRED_MANY) {
|
| 1332 | folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
|
| 1333 | mpoli->nid,
|
| 1334 | nodemask);
|
| 1335 |
|
| 1336 |
|
| 1337 | nodemask = NULL;
|
| 1338 | }
|
| 1339 |
|
| 1340 | if (!folio) {
|
| 1341 | folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
|
| 1342 | mpoli->nid,
|
| 1343 | nodemask);
|
| 1344 | }
|
| 1345 | return folio;
|
| 1346 | }
|
| 1347 |
|
| 1348 | #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC)
|
| 1349 | static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask,
|
| 1350 | int nid, nodemask_t *nodemask)
|
| 1351 | {
|
| 1352 | struct folio *folio;
|
| 1353 |
|
| 1354 | folio = hugetlb_cma_alloc_frozen_folio(order, gfp_mask, nid, nodemask);
|
| 1355 | if (folio)
|
| 1356 | return folio;
|
| 1357 |
|
| 1358 | if (hugetlb_cma_exclusive_alloc())
|
| 1359 | return NULL;
|
| 1360 |
|
| 1361 | folio = (struct folio *)alloc_contig_frozen_pages(1 << order, gfp_mask,
|
| 1362 | nid, nodemask);
|
| 1363 | return folio;
|
| 1364 | }
|
| 1365 | #else
|
| 1366 | static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask, int nid,
|
| 1367 | nodemask_t *nodemask)
|
| 1368 | {
|
| 1369 | return NULL;
|
| 1370 | }
|
| 1371 | #endif
|
| 1372 |
|
| 1373 | |
| 1374 | * Remove hugetlb folio from lists.
|
| 1375 | * If vmemmap exists for the folio, clear the hugetlb flag so that the
|
| 1376 | * folio appears as just a compound page. Otherwise, wait until after
|
| 1377 | * allocating vmemmap to clear the flag.
|
| 1378 | *
|
| 1379 | * Must be called with hugetlb lock held.
|
| 1380 |
|
| 1381 | void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
|
| 1382 | bool adjust_surplus)
|
| 1383 | {
|
| 1384 | int nid = folio_nid(folio);
|
| 1385 |
|
| 1386 | VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
|
| 1387 | VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
|
| 1388 |
|
| 1389 | lockdep_assert_held(&hugetlb_lock);
|
| 1390 | if (hstate_is_gigantic_no_runtime(h))
|
| 1391 | return;
|
| 1392 |
|
| 1393 | list_del(&folio->lru);
|
| 1394 |
|
| 1395 | if (folio_test_hugetlb_freed(folio)) {
|
| 1396 | folio_clear_hugetlb_freed(folio);
|
| 1397 | h->free_huge_pages--;
|
| 1398 | h->free_huge_pages_node[nid]--;
|
| 1399 | }
|
| 1400 | if (adjust_surplus) {
|
| 1401 | h->surplus_huge_pages--;
|
| 1402 | h->surplus_huge_pages_node[nid]--;
|
| 1403 | }
|
| 1404 |
|
| 1405 | |
| 1406 | * We can only clear the hugetlb flag after allocating vmemmap
|
| 1407 | * pages. Otherwise, someone (memory error handling) may try to write
|
| 1408 | * to tail struct pages.
|
| 1409 |
|
| 1410 | if (!folio_test_hugetlb_vmemmap_optimized(folio))
|
| 1411 | __folio_clear_hugetlb(folio);
|
| 1412 |
|
| 1413 | h->nr_huge_pages--;
|
| 1414 | h->nr_huge_pages_node[nid]--;
|
| 1415 | }
|
| 1416 |
|
| 1417 | void add_hugetlb_folio(struct hstate *h, struct folio *folio,
|
| 1418 | bool adjust_surplus)
|
| 1419 | {
|
| 1420 | int nid = folio_nid(folio);
|
| 1421 |
|
| 1422 | VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
|
| 1423 |
|
| 1424 | lockdep_assert_held(&hugetlb_lock);
|
| 1425 |
|
| 1426 | INIT_LIST_HEAD(&folio->lru);
|
| 1427 | h->nr_huge_pages++;
|
| 1428 | h->nr_huge_pages_node[nid]++;
|
| 1429 |
|
| 1430 | if (adjust_surplus) {
|
| 1431 | h->surplus_huge_pages++;
|
| 1432 | h->surplus_huge_pages_node[nid]++;
|
| 1433 | }
|
| 1434 |
|
| 1435 | __folio_set_hugetlb(folio);
|
| 1436 | folio_change_private(folio, NULL);
|
| 1437 | |
| 1438 | * We have to set hugetlb_vmemmap_optimized again as above
|
| 1439 | * folio_change_private(folio, NULL) cleared it.
|
| 1440 |
|
| 1441 | folio_set_hugetlb_vmemmap_optimized(folio);
|
| 1442 |
|
| 1443 | arch_clear_hugetlb_flags(folio);
|
| 1444 | enqueue_hugetlb_folio(h, folio);
|
| 1445 | }
|
| 1446 |
|
| 1447 | static void __update_and_free_hugetlb_folio(struct hstate *h,
|
| 1448 | struct folio *folio)
|
| 1449 | {
|
| 1450 | bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio);
|
| 1451 |
|
| 1452 | if (hstate_is_gigantic_no_runtime(h))
|
| 1453 | return;
|
| 1454 |
|
| 1455 | |
| 1456 | * If we don't know which subpages are hwpoisoned, we can't free
|
| 1457 | * the hugepage, so it's leaked intentionally.
|
| 1458 |
|
| 1459 | if (folio_test_hugetlb_raw_hwp_unreliable(folio))
|
| 1460 | return;
|
| 1461 |
|
| 1462 | |
| 1463 | * If folio is not vmemmap optimized (!clear_flag), then the folio
|
| 1464 | * is no longer identified as a hugetlb page. hugetlb_vmemmap_restore_folio
|
| 1465 | * can only be passed hugetlb pages and will BUG otherwise.
|
| 1466 |
|
| 1467 | if (clear_flag && hugetlb_vmemmap_restore_folio(h, folio)) {
|
| 1468 | spin_lock_irq(&hugetlb_lock);
|
| 1469 | |
| 1470 | * If we cannot allocate vmemmap pages, just refuse to free the
|
| 1471 | * page and put the page back on the hugetlb free list and treat
|
| 1472 | * as a surplus page.
|
| 1473 |
|
| 1474 | add_hugetlb_folio(h, folio, true);
|
| 1475 | spin_unlock_irq(&hugetlb_lock);
|
| 1476 | return;
|
| 1477 | }
|
| 1478 |
|
| 1479 | |
| 1480 | * If vmemmap pages were allocated above, then we need to clear the
|
| 1481 | * hugetlb flag under the hugetlb lock.
|
| 1482 |
|
| 1483 | if (folio_test_hugetlb(folio)) {
|
| 1484 | spin_lock_irq(&hugetlb_lock);
|
| 1485 | __folio_clear_hugetlb(folio);
|
| 1486 | spin_unlock_irq(&hugetlb_lock);
|
| 1487 | }
|
| 1488 |
|
| 1489 | |
| 1490 | * Move PageHWPoison flag from head page to the raw error pages,
|
| 1491 | * which makes any healthy subpages reusable.
|
| 1492 |
|
| 1493 | if (unlikely(folio_test_hwpoison(folio)))
|
| 1494 | folio_clear_hugetlb_hwpoison(folio);
|
| 1495 |
|
| 1496 | VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
|
| 1497 | if (folio_test_hugetlb_cma(folio))
|
| 1498 | hugetlb_cma_free_frozen_folio(folio);
|
| 1499 | else
|
| 1500 | free_frozen_pages(&folio->page, folio_order(folio));
|
| 1501 | }
|
| 1502 |
|
| 1503 | |
| 1504 | * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
|
| 1505 | * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
|
| 1506 | * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
|
| 1507 | * the vmemmap pages.
|
| 1508 | *
|
| 1509 | * free_hpage_workfn() locklessly retrieves the linked list of pages to be
|
| 1510 | * freed and frees them one-by-one. As the page->mapping pointer is going
|
| 1511 | * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
|
| 1512 | * structure of a lockless linked list of huge pages to be freed.
|
| 1513 |
|
| 1514 | static LLIST_HEAD(hpage_freelist);
|
| 1515 |
|
| 1516 | static void free_hpage_workfn(struct work_struct *work)
|
| 1517 | {
|
| 1518 | struct llist_node *node;
|
| 1519 |
|
| 1520 | node = llist_del_all(&hpage_freelist);
|
| 1521 |
|
| 1522 | while (node) {
|
| 1523 | struct folio *folio;
|
| 1524 | struct hstate *h;
|
| 1525 |
|
| 1526 | folio = container_of((struct address_space **)node,
|
| 1527 | struct folio, mapping);
|
| 1528 | node = node->next;
|
| 1529 | folio->mapping = NULL;
|
| 1530 | |
| 1531 | * The VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio) in
|
| 1532 | * folio_hstate() is going to trigger because a previous call to
|
| 1533 | * remove_hugetlb_folio() will clear the hugetlb bit, so do
|
| 1534 | * not use folio_hstate() directly.
|
| 1535 |
|
| 1536 | h = size_to_hstate(folio_size(folio));
|
| 1537 |
|
| 1538 | __update_and_free_hugetlb_folio(h, folio);
|
| 1539 |
|
| 1540 | cond_resched();
|
| 1541 | }
|
| 1542 | }
|
| 1543 | static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
|
| 1544 |
|
| 1545 | static inline void flush_free_hpage_work(struct hstate *h)
|
| 1546 | {
|
| 1547 | if (hugetlb_vmemmap_optimizable(h))
|
| 1548 | flush_work(&free_hpage_work);
|
| 1549 | }
|
| 1550 |
|
| 1551 | static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
|
| 1552 | bool atomic)
|
| 1553 | {
|
| 1554 | if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
|
| 1555 | __update_and_free_hugetlb_folio(h, folio);
|
| 1556 | return;
|
| 1557 | }
|
| 1558 |
|
| 1559 | |
| 1560 | * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
|
| 1561 | *
|
| 1562 | * Only call schedule_work() if hpage_freelist is previously
|
| 1563 | * empty. Otherwise, schedule_work() had been called but the workfn
|
| 1564 | * hasn't retrieved the list yet.
|
| 1565 |
|
| 1566 | if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
|
| 1567 | schedule_work(&free_hpage_work);
|
| 1568 | }
|
| 1569 |
|
| 1570 | static void bulk_vmemmap_restore_error(struct hstate *h,
|
| 1571 | struct list_head *folio_list,
|
| 1572 | struct list_head *non_hvo_folios)
|
| 1573 | {
|
| 1574 | struct folio *folio, *t_folio;
|
| 1575 |
|
| 1576 | if (!list_empty(non_hvo_folios)) {
|
| 1577 | |
| 1578 | * Free any restored hugetlb pages so that restore of the
|
| 1579 | * entire list can be retried.
|
| 1580 | * The idea is that in the common case of ENOMEM errors freeing
|
| 1581 | * hugetlb pages with vmemmap we will free up memory so that we
|
| 1582 | * can allocate vmemmap for more hugetlb pages.
|
| 1583 |
|
| 1584 | list_for_each_entry_safe(folio, t_folio, non_hvo_folios, lru) {
|
| 1585 | list_del(&folio->lru);
|
| 1586 | spin_lock_irq(&hugetlb_lock);
|
| 1587 | __folio_clear_hugetlb(folio);
|
| 1588 | spin_unlock_irq(&hugetlb_lock);
|
| 1589 | update_and_free_hugetlb_folio(h, folio, false);
|
| 1590 | cond_resched();
|
| 1591 | }
|
| 1592 | } else {
|
| 1593 | |
| 1594 | * In the case where there are no folios which can be
|
| 1595 | * immediately freed, we loop through the list trying to restore
|
| 1596 | * vmemmap individually in the hope that someone elsewhere may
|
| 1597 | * have done something to cause success (such as freeing some
|
| 1598 | * memory). If unable to restore a hugetlb page, the hugetlb
|
| 1599 | * page is made a surplus page and removed from the list.
|
| 1600 | * If are able to restore vmemmap and free one hugetlb page, we
|
| 1601 | * quit processing the list to retry the bulk operation.
|
| 1602 |
|
| 1603 | list_for_each_entry_safe(folio, t_folio, folio_list, lru)
|
| 1604 | if (hugetlb_vmemmap_restore_folio(h, folio)) {
|
| 1605 | list_del(&folio->lru);
|
| 1606 | spin_lock_irq(&hugetlb_lock);
|
| 1607 | add_hugetlb_folio(h, folio, true);
|
| 1608 | spin_unlock_irq(&hugetlb_lock);
|
| 1609 | } else {
|
| 1610 | list_del(&folio->lru);
|
| 1611 | spin_lock_irq(&hugetlb_lock);
|
| 1612 | __folio_clear_hugetlb(folio);
|
| 1613 | spin_unlock_irq(&hugetlb_lock);
|
| 1614 | update_and_free_hugetlb_folio(h, folio, false);
|
| 1615 | cond_resched();
|
| 1616 | break;
|
| 1617 | }
|
| 1618 | }
|
| 1619 | }
|
| 1620 |
|
| 1621 | static void update_and_free_pages_bulk(struct hstate *h,
|
| 1622 | struct list_head *folio_list)
|
| 1623 | {
|
| 1624 | long ret;
|
| 1625 | struct folio *folio, *t_folio;
|
| 1626 | LIST_HEAD(non_hvo_folios);
|
| 1627 |
|
| 1628 | |
| 1629 | * First allocate required vmemmmap (if necessary) for all folios.
|
| 1630 | * Carefully handle errors and free up any available hugetlb pages
|
| 1631 | * in an effort to make forward progress.
|
| 1632 |
|
| 1633 | retry:
|
| 1634 | ret = hugetlb_vmemmap_restore_folios(h, folio_list, &non_hvo_folios);
|
| 1635 | if (ret < 0) {
|
| 1636 | bulk_vmemmap_restore_error(h, folio_list, &non_hvo_folios);
|
| 1637 | goto retry;
|
| 1638 | }
|
| 1639 |
|
| 1640 | |
| 1641 | * At this point, list should be empty, ret should be >= 0 and there
|
| 1642 | * should only be pages on the non_hvo_folios list.
|
| 1643 | * Do note that the non_hvo_folios list could be empty.
|
| 1644 | * Without HVO enabled, ret will be 0 and there is no need to call
|
| 1645 | * __folio_clear_hugetlb as this was done previously.
|
| 1646 |
|
| 1647 | VM_WARN_ON(!list_empty(folio_list));
|
| 1648 | VM_WARN_ON(ret < 0);
|
| 1649 | if (!list_empty(&non_hvo_folios) && ret) {
|
| 1650 | spin_lock_irq(&hugetlb_lock);
|
| 1651 | list_for_each_entry(folio, &non_hvo_folios, lru)
|
| 1652 | __folio_clear_hugetlb(folio);
|
| 1653 | spin_unlock_irq(&hugetlb_lock);
|
| 1654 | }
|
| 1655 |
|
| 1656 | list_for_each_entry_safe(folio, t_folio, &non_hvo_folios, lru) {
|
| 1657 | update_and_free_hugetlb_folio(h, folio, false);
|
| 1658 | cond_resched();
|
| 1659 | }
|
| 1660 | }
|
| 1661 |
|
| 1662 | struct hstate *size_to_hstate(unsigned long size)
|
| 1663 | {
|
| 1664 | struct hstate *h;
|
| 1665 |
|
| 1666 | for_each_hstate(h) {
|
| 1667 | if (huge_page_size(h) == size)
|
| 1668 | return h;
|
| 1669 | }
|
| 1670 | return NULL;
|
| 1671 | }
|
| 1672 |
|
| 1673 | void free_huge_folio(struct folio *folio)
|
| 1674 | {
|
| 1675 | |
| 1676 | * Can't pass hstate in here because it is called from the
|
| 1677 | * generic mm code.
|
| 1678 |
|
| 1679 | struct hstate *h = folio_hstate(folio);
|
| 1680 | int nid = folio_nid(folio);
|
| 1681 | struct hugepage_subpool *spool = hugetlb_folio_subpool(folio);
|
| 1682 | bool restore_reserve;
|
| 1683 | unsigned long flags;
|
| 1684 |
|
| 1685 | VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
|
| 1686 | VM_BUG_ON_FOLIO(folio_mapcount(folio), folio);
|
| 1687 |
|
| 1688 | hugetlb_set_folio_subpool(folio, NULL);
|
| 1689 | if (folio_test_anon(folio))
|
| 1690 | __ClearPageAnonExclusive(&folio->page);
|
| 1691 | folio->mapping = NULL;
|
| 1692 | restore_reserve = folio_test_hugetlb_restore_reserve(folio);
|
| 1693 | folio_clear_hugetlb_restore_reserve(folio);
|
| 1694 |
|
| 1695 | |
| 1696 | * If HPageRestoreReserve was set on page, page allocation consumed a
|
| 1697 | * reservation. If the page was associated with a subpool, there
|
| 1698 | * would have been a page reserved in the subpool before allocation
|
| 1699 | * via hugepage_subpool_get_pages(). Since we are 'restoring' the
|
| 1700 | * reservation, do not call hugepage_subpool_put_pages() as this will
|
| 1701 | * remove the reserved page from the subpool.
|
| 1702 |
|
| 1703 | if (!restore_reserve) {
|
| 1704 | |
| 1705 | * A return code of zero implies that the subpool will be
|
| 1706 | * under its minimum size if the reservation is not restored
|
| 1707 | * after page is free. Therefore, force restore_reserve
|
| 1708 | * operation.
|
| 1709 |
|
| 1710 | if (hugepage_subpool_put_pages(spool, 1) == 0)
|
| 1711 | restore_reserve = true;
|
| 1712 | }
|
| 1713 |
|
| 1714 | spin_lock_irqsave(&hugetlb_lock, flags);
|
| 1715 | folio_clear_hugetlb_migratable(folio);
|
| 1716 | hugetlb_cgroup_uncharge_folio(hstate_index(h),
|
| 1717 | pages_per_huge_page(h), folio);
|
| 1718 | hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
|
| 1719 | pages_per_huge_page(h), folio);
|
| 1720 | lruvec_stat_mod_folio(folio, NR_HUGETLB, -pages_per_huge_page(h));
|
| 1721 | mem_cgroup_uncharge(folio);
|
| 1722 | if (restore_reserve)
|
| 1723 | h->resv_huge_pages++;
|
| 1724 |
|
| 1725 | if (folio_test_hugetlb_temporary(folio)) {
|
| 1726 | remove_hugetlb_folio(h, folio, false);
|
| 1727 | spin_unlock_irqrestore(&hugetlb_lock, flags);
|
| 1728 | update_and_free_hugetlb_folio(h, folio, true);
|
| 1729 | } else if (h->surplus_huge_pages_node[nid]) {
|
| 1730 |
|
| 1731 | remove_hugetlb_folio(h, folio, true);
|
| 1732 | spin_unlock_irqrestore(&hugetlb_lock, flags);
|
| 1733 | update_and_free_hugetlb_folio(h, folio, true);
|
| 1734 | } else {
|
| 1735 | arch_clear_hugetlb_flags(folio);
|
| 1736 | enqueue_hugetlb_folio(h, folio);
|
| 1737 | spin_unlock_irqrestore(&hugetlb_lock, flags);
|
| 1738 | }
|
| 1739 | }
|
| 1740 |
|
| 1741 | |
| 1742 | * Must be called with the hugetlb lock held
|
| 1743 |
|
| 1744 | static void account_new_hugetlb_folio(struct hstate *h, struct folio *folio)
|
| 1745 | {
|
| 1746 | lockdep_assert_held(&hugetlb_lock);
|
| 1747 | h->nr_huge_pages++;
|
| 1748 | h->nr_huge_pages_node[folio_nid(folio)]++;
|
| 1749 | }
|
| 1750 |
|
| 1751 | void init_new_hugetlb_folio(struct folio *folio)
|
| 1752 | {
|
| 1753 | __folio_set_hugetlb(folio);
|
| 1754 | INIT_LIST_HEAD(&folio->lru);
|
| 1755 | hugetlb_set_folio_subpool(folio, NULL);
|
| 1756 | set_hugetlb_cgroup(folio, NULL);
|
| 1757 | set_hugetlb_cgroup_rsvd(folio, NULL);
|
| 1758 | }
|
| 1759 |
|
| 1760 | |
| 1761 | * Find and lock address space (mapping) in write mode.
|
| 1762 | *
|
| 1763 | * Upon entry, the folio is locked which means that folio_mapping() is
|
| 1764 | * stable. Due to locking order, we can only trylock_write. If we can
|
| 1765 | * not get the lock, simply return NULL to caller.
|
| 1766 |
|
| 1767 | struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio)
|
| 1768 | {
|
| 1769 | struct address_space *mapping = folio_mapping(folio);
|
| 1770 |
|
| 1771 | if (!mapping)
|
| 1772 | return mapping;
|
| 1773 |
|
| 1774 | if (i_mmap_trylock_write(mapping))
|
| 1775 | return mapping;
|
| 1776 |
|
| 1777 | return NULL;
|
| 1778 | }
|
| 1779 |
|
| 1780 | static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
|
| 1781 | int nid, nodemask_t *nmask, nodemask_t *node_alloc_noretry)
|
| 1782 | {
|
| 1783 | struct folio *folio;
|
| 1784 | bool alloc_try_hard = true;
|
| 1785 |
|
| 1786 | |
| 1787 | * By default we always try hard to allocate the folio with
|
| 1788 | * __GFP_RETRY_MAYFAIL flag. However, if we are allocating folios in
|
| 1789 | * a loop (to adjust global huge page counts) and previous allocation
|
| 1790 | * failed, do not continue to try hard on the same node. Use the
|
| 1791 | * node_alloc_noretry bitmap to manage this state information.
|
| 1792 |
|
| 1793 | if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
|
| 1794 | alloc_try_hard = false;
|
| 1795 | if (alloc_try_hard)
|
| 1796 | gfp_mask |= __GFP_RETRY_MAYFAIL;
|
| 1797 |
|
| 1798 | folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask,
|
| 1799 | ALLOC_DEFAULT);
|
| 1800 |
|
| 1801 | |
| 1802 | * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
|
| 1803 | * folio this indicates an overall state change. Clear bit so
|
| 1804 | * that we resume normal 'try hard' allocations.
|
| 1805 |
|
| 1806 | if (node_alloc_noretry && folio && !alloc_try_hard)
|
| 1807 | node_clear(nid, *node_alloc_noretry);
|
| 1808 |
|
| 1809 | |
| 1810 | * If we tried hard to get a folio but failed, set bit so that
|
| 1811 | * subsequent attempts will not try as hard until there is an
|
| 1812 | * overall state change.
|
| 1813 |
|
| 1814 | if (node_alloc_noretry && !folio && alloc_try_hard)
|
| 1815 | node_set(nid, *node_alloc_noretry);
|
| 1816 |
|
| 1817 | if (!folio) {
|
| 1818 | __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
|
| 1819 | return NULL;
|
| 1820 | }
|
| 1821 |
|
| 1822 | __count_vm_event(HTLB_BUDDY_PGALLOC);
|
| 1823 | return folio;
|
| 1824 | }
|
| 1825 |
|
| 1826 | static struct folio *only_alloc_fresh_hugetlb_folio(struct hstate *h,
|
| 1827 | gfp_t gfp_mask, int nid, nodemask_t *nmask,
|
| 1828 | nodemask_t *node_alloc_noretry)
|
| 1829 | {
|
| 1830 | struct folio *folio;
|
| 1831 | int order = huge_page_order(h);
|
| 1832 |
|
| 1833 | if (nid == NUMA_NO_NODE)
|
| 1834 | nid = numa_mem_id();
|
| 1835 |
|
| 1836 | if (order_is_gigantic(order))
|
| 1837 | folio = alloc_gigantic_frozen_folio(order, gfp_mask, nid, nmask);
|
| 1838 | else
|
| 1839 | folio = alloc_buddy_frozen_folio(order, gfp_mask, nid, nmask,
|
| 1840 | node_alloc_noretry);
|
| 1841 | if (folio)
|
| 1842 | init_new_hugetlb_folio(folio);
|
| 1843 | return folio;
|
| 1844 | }
|
| 1845 |
|
| 1846 | |
| 1847 | * Common helper to allocate a fresh hugetlb folio. All specific allocators
|
| 1848 | * should use this function to get new hugetlb folio
|
| 1849 | *
|
| 1850 | * Note that returned folio is 'frozen': ref count of head page and all tail
|
| 1851 | * pages is zero, and the accounting must be done in the caller.
|
| 1852 |
|
| 1853 | static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
|
| 1854 | gfp_t gfp_mask, int nid, nodemask_t *nmask)
|
| 1855 | {
|
| 1856 | struct folio *folio;
|
| 1857 |
|
| 1858 | folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
|
| 1859 | if (folio)
|
| 1860 | hugetlb_vmemmap_optimize_folio(h, folio);
|
| 1861 | return folio;
|
| 1862 | }
|
| 1863 |
|
| 1864 | void prep_and_add_allocated_folios(struct hstate *h,
|
| 1865 | struct list_head *folio_list)
|
| 1866 | {
|
| 1867 | unsigned long flags;
|
| 1868 | struct folio *folio, *tmp_f;
|
| 1869 |
|
| 1870 |
|
| 1871 | hugetlb_vmemmap_optimize_folios(h, folio_list);
|
| 1872 |
|
| 1873 |
|
| 1874 | spin_lock_irqsave(&hugetlb_lock, flags);
|
| 1875 | list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
|
| 1876 | account_new_hugetlb_folio(h, folio);
|
| 1877 | enqueue_hugetlb_folio(h, folio);
|
| 1878 | }
|
| 1879 | spin_unlock_irqrestore(&hugetlb_lock, flags);
|
| 1880 | }
|
| 1881 |
|
| 1882 | |
| 1883 | * Allocates a fresh hugetlb page in a node interleaved manner. The page
|
| 1884 | * will later be added to the appropriate hugetlb pool.
|
| 1885 |
|
| 1886 | static struct folio *alloc_pool_huge_folio(struct hstate *h,
|
| 1887 | nodemask_t *nodes_allowed,
|
| 1888 | nodemask_t *node_alloc_noretry,
|
| 1889 | int *next_node)
|
| 1890 | {
|
| 1891 | gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
|
| 1892 | int nr_nodes, node;
|
| 1893 |
|
| 1894 | for_each_node_mask_to_alloc(next_node, nr_nodes, node, nodes_allowed) {
|
| 1895 | struct folio *folio;
|
| 1896 |
|
| 1897 | folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, node,
|
| 1898 | nodes_allowed, node_alloc_noretry);
|
| 1899 | if (folio)
|
| 1900 | return folio;
|
| 1901 | }
|
| 1902 |
|
| 1903 | return NULL;
|
| 1904 | }
|
| 1905 |
|
| 1906 | |
| 1907 | * Remove huge page from pool from next node to free. Attempt to keep
|
| 1908 | * persistent huge pages more or less balanced over allowed nodes.
|
| 1909 | * This routine only 'removes' the hugetlb page. The caller must make
|
| 1910 | * an additional call to free the page to low level allocators.
|
| 1911 | * Called with hugetlb_lock locked.
|
| 1912 |
|
| 1913 | static struct folio *remove_pool_hugetlb_folio(struct hstate *h,
|
| 1914 | nodemask_t *nodes_allowed, bool acct_surplus)
|
| 1915 | {
|
| 1916 | int nr_nodes, node;
|
| 1917 | struct folio *folio = NULL;
|
| 1918 |
|
| 1919 | lockdep_assert_held(&hugetlb_lock);
|
| 1920 | for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
|
| 1921 | |
| 1922 | * If we're returning unused surplus pages, only examine
|
| 1923 | * nodes with surplus pages.
|
| 1924 |
|
| 1925 | if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
|
| 1926 | !list_empty(&h->hugepage_freelists[node])) {
|
| 1927 | folio = list_entry(h->hugepage_freelists[node].next,
|
| 1928 | struct folio, lru);
|
| 1929 | remove_hugetlb_folio(h, folio, acct_surplus);
|
| 1930 | break;
|
| 1931 | }
|
| 1932 | }
|
| 1933 |
|
| 1934 | return folio;
|
| 1935 | }
|
| 1936 |
|
| 1937 | |
| 1938 | * Dissolve a given free hugetlb folio into free buddy pages. This function
|
| 1939 | * does nothing for in-use hugetlb folios and non-hugetlb folios.
|
| 1940 | * This function returns values like below:
|
| 1941 | *
|
| 1942 | * -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
|
| 1943 | * when the system is under memory pressure and the feature of
|
| 1944 | * freeing unused vmemmap pages associated with each hugetlb page
|
| 1945 | * is enabled.
|
| 1946 | * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
|
| 1947 | * (allocated or reserved.)
|
| 1948 | * 0: successfully dissolved free hugepages or the page is not a
|
| 1949 | * hugepage (considered as already dissolved)
|
| 1950 |
|
| 1951 | int dissolve_free_hugetlb_folio(struct folio *folio)
|
| 1952 | {
|
| 1953 | int rc = -EBUSY;
|
| 1954 |
|
| 1955 | retry:
|
| 1956 |
|
| 1957 | if (!folio_test_hugetlb(folio))
|
| 1958 | return 0;
|
| 1959 |
|
| 1960 | spin_lock_irq(&hugetlb_lock);
|
| 1961 | if (!folio_test_hugetlb(folio)) {
|
| 1962 | rc = 0;
|
| 1963 | goto out;
|
| 1964 | }
|
| 1965 |
|
| 1966 | if (!folio_ref_count(folio)) {
|
| 1967 | struct hstate *h = folio_hstate(folio);
|
| 1968 | bool adjust_surplus = false;
|
| 1969 |
|
| 1970 | |
| 1971 | * remove_hugetlb_folio()/update_and_free_hugetlb_folio() bail
|
| 1972 | * for gigantic hstates without runtime support, so dissolving one
|
| 1973 | * here would leave it on the free list and, on vmemmap restore
|
| 1974 | * failure, the add_hugetlb_folio() rollback corrupts that list.
|
| 1975 |
|
| 1976 | if (hstate_is_gigantic_no_runtime(h))
|
| 1977 | goto out;
|
| 1978 |
|
| 1979 | if (!available_huge_pages(h))
|
| 1980 | goto out;
|
| 1981 |
|
| 1982 | |
| 1983 | * We should make sure that the page is already on the free list
|
| 1984 | * when it is dissolved.
|
| 1985 |
|
| 1986 | if (unlikely(!folio_test_hugetlb_freed(folio))) {
|
| 1987 | spin_unlock_irq(&hugetlb_lock);
|
| 1988 | cond_resched();
|
| 1989 |
|
| 1990 | |
| 1991 | * Theoretically, we should return -EBUSY when we
|
| 1992 | * encounter this race. In fact, we have a chance
|
| 1993 | * to successfully dissolve the page if we do a
|
| 1994 | * retry. Because the race window is quite small.
|
| 1995 | * If we seize this opportunity, it is an optimization
|
| 1996 | * for increasing the success rate of dissolving page.
|
| 1997 |
|
| 1998 | goto retry;
|
| 1999 | }
|
| 2000 |
|
| 2001 | if (h->surplus_huge_pages_node[folio_nid(folio)])
|
| 2002 | adjust_surplus = true;
|
| 2003 | remove_hugetlb_folio(h, folio, adjust_surplus);
|
| 2004 | if (!adjust_surplus)
|
| 2005 | h->max_huge_pages--;
|
| 2006 | spin_unlock_irq(&hugetlb_lock);
|
| 2007 |
|
| 2008 | |
| 2009 | * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
|
| 2010 | * before freeing the page. update_and_free_hugtlb_folio will fail to
|
| 2011 | * free the page if it can not allocate required vmemmap. We
|
| 2012 | * need to adjust max_huge_pages if the page is not freed.
|
| 2013 | * Attempt to allocate vmemmmap here so that we can take
|
| 2014 | * appropriate action on failure.
|
| 2015 | *
|
| 2016 | * The folio_test_hugetlb check here is because
|
| 2017 | * remove_hugetlb_folio will clear hugetlb folio flag for
|
| 2018 | * non-vmemmap optimized hugetlb folios.
|
| 2019 |
|
| 2020 | if (folio_test_hugetlb(folio)) {
|
| 2021 | rc = hugetlb_vmemmap_restore_folio(h, folio);
|
| 2022 | if (rc) {
|
| 2023 | spin_lock_irq(&hugetlb_lock);
|
| 2024 | add_hugetlb_folio(h, folio, adjust_surplus);
|
| 2025 | if (!adjust_surplus)
|
| 2026 | h->max_huge_pages++;
|
| 2027 | goto out;
|
| 2028 | }
|
| 2029 | } else {
|
| 2030 | rc = 0;
|
| 2031 | }
|
| 2032 |
|
| 2033 | update_and_free_hugetlb_folio(h, folio, false);
|
| 2034 | return rc;
|
| 2035 | }
|
| 2036 | out:
|
| 2037 | spin_unlock_irq(&hugetlb_lock);
|
| 2038 | return rc;
|
| 2039 | }
|
| 2040 |
|
| 2041 | |
| 2042 | * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
|
| 2043 | * make specified memory blocks removable from the system.
|
| 2044 | * Note that this will dissolve a free gigantic hugepage completely, if any
|
| 2045 | * part of it lies within the given range.
|
| 2046 | * Also note that if dissolve_free_hugetlb_folio() returns with an error, all
|
| 2047 | * free hugetlb folios that were dissolved before that error are lost.
|
| 2048 |
|
| 2049 | int dissolve_free_hugetlb_folios(unsigned long start_pfn, unsigned long end_pfn)
|
| 2050 | {
|
| 2051 | unsigned long pfn;
|
| 2052 | struct folio *folio;
|
| 2053 | int rc = 0;
|
| 2054 | unsigned int order;
|
| 2055 | struct hstate *h;
|
| 2056 |
|
| 2057 | if (!hugepages_supported())
|
| 2058 | return rc;
|
| 2059 |
|
| 2060 | order = huge_page_order(&default_hstate);
|
| 2061 | for_each_hstate(h)
|
| 2062 | order = min(order, huge_page_order(h));
|
| 2063 |
|
| 2064 | for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) {
|
| 2065 | folio = pfn_folio(pfn);
|
| 2066 | rc = dissolve_free_hugetlb_folio(folio);
|
| 2067 | if (rc)
|
| 2068 | break;
|
| 2069 | }
|
| 2070 |
|
| 2071 | return rc;
|
| 2072 | }
|
| 2073 |
|
| 2074 | |
| 2075 | * Allocates a fresh surplus page from the page allocator.
|
| 2076 |
|
| 2077 | static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h,
|
| 2078 | gfp_t gfp_mask, int nid, nodemask_t *nmask)
|
| 2079 | {
|
| 2080 | struct folio *folio = NULL;
|
| 2081 |
|
| 2082 | if (hstate_is_gigantic_no_runtime(h))
|
| 2083 | return NULL;
|
| 2084 |
|
| 2085 | spin_lock_irq(&hugetlb_lock);
|
| 2086 | if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
|
| 2087 | goto out_unlock;
|
| 2088 | spin_unlock_irq(&hugetlb_lock);
|
| 2089 |
|
| 2090 | folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask);
|
| 2091 | if (!folio)
|
| 2092 | return NULL;
|
| 2093 |
|
| 2094 | spin_lock_irq(&hugetlb_lock);
|
| 2095 | |
| 2096 | * nr_huge_pages needs to be adjusted within the same lock cycle
|
| 2097 | * as surplus_pages, otherwise it might confuse
|
| 2098 | * persistent_huge_pages() momentarily.
|
| 2099 |
|
| 2100 | account_new_hugetlb_folio(h, folio);
|
| 2101 |
|
| 2102 | |
| 2103 | * We could have raced with the pool size change.
|
| 2104 | * Double check that and simply deallocate the new page
|
| 2105 | * if we would end up overcommiting the surpluses. Abuse
|
| 2106 | * temporary page to workaround the nasty free_huge_folio
|
| 2107 | * codeflow
|
| 2108 |
|
| 2109 | if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
|
| 2110 | folio_set_hugetlb_temporary(folio);
|
| 2111 | spin_unlock_irq(&hugetlb_lock);
|
| 2112 | free_huge_folio(folio);
|
| 2113 | return NULL;
|
| 2114 | }
|
| 2115 |
|
| 2116 | h->surplus_huge_pages++;
|
| 2117 | h->surplus_huge_pages_node[folio_nid(folio)]++;
|
| 2118 |
|
| 2119 | out_unlock:
|
| 2120 | spin_unlock_irq(&hugetlb_lock);
|
| 2121 |
|
| 2122 | return folio;
|
| 2123 | }
|
| 2124 |
|
| 2125 | static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
|
| 2126 | int nid, nodemask_t *nmask)
|
| 2127 | {
|
| 2128 | struct folio *folio;
|
| 2129 |
|
| 2130 | if (hstate_is_gigantic(h))
|
| 2131 | return NULL;
|
| 2132 |
|
| 2133 | folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask);
|
| 2134 | if (!folio)
|
| 2135 | return NULL;
|
| 2136 |
|
| 2137 | spin_lock_irq(&hugetlb_lock);
|
| 2138 | account_new_hugetlb_folio(h, folio);
|
| 2139 | spin_unlock_irq(&hugetlb_lock);
|
| 2140 |
|
| 2141 |
|
| 2142 | folio_ref_unfreeze(folio, 1);
|
| 2143 | |
| 2144 | * We do not account these pages as surplus because they are only
|
| 2145 | * temporary and will be released properly on the last reference
|
| 2146 |
|
| 2147 | folio_set_hugetlb_temporary(folio);
|
| 2148 |
|
| 2149 | return folio;
|
| 2150 | }
|
| 2151 |
|
| 2152 | static
|
| 2153 | struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
|
| 2154 | gfp_t gfp_mask, struct mempolicy_interpreted *mpoli)
|
| 2155 | {
|
| 2156 | struct folio *folio = NULL;
|
| 2157 | nodemask_t *nodemask = mpoli->nodemask;
|
| 2158 |
|
| 2159 | if (mpoli->mode == MPOL_PREFERRED_MANY) {
|
| 2160 | gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
|
| 2161 |
|
| 2162 | folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid,
|
| 2163 | nodemask);
|
| 2164 |
|
| 2165 |
|
| 2166 | nodemask = NULL;
|
| 2167 | }
|
| 2168 |
|
| 2169 | if (!folio) {
|
| 2170 | folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid,
|
| 2171 | nodemask);
|
| 2172 | }
|
| 2173 |
|
| 2174 | return folio;
|
| 2175 | }
|
| 2176 |
|
| 2177 | struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
|
| 2178 | nodemask_t *nmask, gfp_t gfp_mask)
|
| 2179 | {
|
| 2180 | struct folio *folio;
|
| 2181 |
|
| 2182 | spin_lock_irq(&hugetlb_lock);
|
| 2183 | if (!h->resv_huge_pages) {
|
| 2184 | spin_unlock_irq(&hugetlb_lock);
|
| 2185 | return NULL;
|
| 2186 | }
|
| 2187 |
|
| 2188 | folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid,
|
| 2189 | nmask);
|
| 2190 | if (folio)
|
| 2191 | h->resv_huge_pages--;
|
| 2192 |
|
| 2193 | spin_unlock_irq(&hugetlb_lock);
|
| 2194 | return folio;
|
| 2195 | }
|
| 2196 |
|
| 2197 |
|
| 2198 | struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
|
| 2199 | nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback)
|
| 2200 | {
|
| 2201 | spin_lock_irq(&hugetlb_lock);
|
| 2202 | if (available_huge_pages(h)) {
|
| 2203 | struct folio *folio;
|
| 2204 |
|
| 2205 | folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
|
| 2206 | preferred_nid, nmask);
|
| 2207 | if (folio) {
|
| 2208 | spin_unlock_irq(&hugetlb_lock);
|
| 2209 | return folio;
|
| 2210 | }
|
| 2211 | }
|
| 2212 | spin_unlock_irq(&hugetlb_lock);
|
| 2213 |
|
| 2214 |
|
| 2215 | if (!allow_alloc_fallback)
|
| 2216 | gfp_mask |= __GFP_THISNODE;
|
| 2217 |
|
| 2218 | return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask);
|
| 2219 | }
|
| 2220 |
|
| 2221 | static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
|
| 2222 | {
|
| 2223 | #ifdef CONFIG_NUMA
|
| 2224 | struct mempolicy *mpol = get_task_policy(current);
|
| 2225 |
|
| 2226 | |
| 2227 | * Only enforce MPOL_BIND policy which overlaps with cpuset policy
|
| 2228 | * (from policy_nodemask) specifically for hugetlb case
|
| 2229 |
|
| 2230 | if (mpol->mode == MPOL_BIND &&
|
| 2231 | (apply_policy_zone(mpol, gfp_zone(gfp)) &&
|
| 2232 | cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
|
| 2233 | return &mpol->nodes;
|
| 2234 | #endif
|
| 2235 | return NULL;
|
| 2236 | }
|
| 2237 |
|
| 2238 | |
| 2239 | * Increase the hugetlb pool such that it can accommodate a reservation
|
| 2240 | * of size 'delta'.
|
| 2241 |
|
| 2242 | static int gather_surplus_pages(struct hstate *h, long delta)
|
| 2243 | __must_hold(&hugetlb_lock)
|
| 2244 | {
|
| 2245 | LIST_HEAD(surplus_list);
|
| 2246 | struct folio *folio, *tmp;
|
| 2247 | int ret;
|
| 2248 | long i;
|
| 2249 | long needed, allocated;
|
| 2250 | bool alloc_ok = true;
|
| 2251 | nodemask_t *mbind_nodemask, alloc_nodemask;
|
| 2252 |
|
| 2253 | mbind_nodemask = policy_mbind_nodemask(htlb_alloc_mask(h));
|
| 2254 | if (mbind_nodemask)
|
| 2255 | nodes_and(alloc_nodemask, *mbind_nodemask, cpuset_current_mems_allowed);
|
| 2256 | else
|
| 2257 | alloc_nodemask = cpuset_current_mems_allowed;
|
| 2258 |
|
| 2259 | lockdep_assert_held(&hugetlb_lock);
|
| 2260 | needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
|
| 2261 | if (needed <= 0) {
|
| 2262 | h->resv_huge_pages += delta;
|
| 2263 | return 0;
|
| 2264 | }
|
| 2265 |
|
| 2266 | allocated = 0;
|
| 2267 |
|
| 2268 | ret = -ENOMEM;
|
| 2269 | retry:
|
| 2270 | spin_unlock_irq(&hugetlb_lock);
|
| 2271 | for (i = 0; i < needed; i++) {
|
| 2272 | folio = NULL;
|
| 2273 |
|
| 2274 | |
| 2275 | * It is okay to use NUMA_NO_NODE because we use numa_mem_id()
|
| 2276 | * down the road to pick the current node if that is the case.
|
| 2277 |
|
| 2278 | folio = alloc_surplus_hugetlb_folio(h, htlb_alloc_mask(h),
|
| 2279 | NUMA_NO_NODE, &alloc_nodemask);
|
| 2280 | if (!folio) {
|
| 2281 | alloc_ok = false;
|
| 2282 | break;
|
| 2283 | }
|
| 2284 | list_add(&folio->lru, &surplus_list);
|
| 2285 | cond_resched();
|
| 2286 | }
|
| 2287 | allocated += i;
|
| 2288 |
|
| 2289 | |
| 2290 | * After retaking hugetlb_lock, we need to recalculate 'needed'
|
| 2291 | * because either resv_huge_pages or free_huge_pages may have changed.
|
| 2292 |
|
| 2293 | spin_lock_irq(&hugetlb_lock);
|
| 2294 | needed = (h->resv_huge_pages + delta) -
|
| 2295 | (h->free_huge_pages + allocated);
|
| 2296 | if (needed > 0) {
|
| 2297 | if (alloc_ok)
|
| 2298 | goto retry;
|
| 2299 | |
| 2300 | * We were not able to allocate enough pages to
|
| 2301 | * satisfy the entire reservation so we free what
|
| 2302 | * we've allocated so far.
|
| 2303 |
|
| 2304 | goto free;
|
| 2305 | }
|
| 2306 | |
| 2307 | * The surplus_list now contains _at_least_ the number of extra pages
|
| 2308 | * needed to accommodate the reservation. Add the appropriate number
|
| 2309 | * of pages to the hugetlb pool and free the extras back to the buddy
|
| 2310 | * allocator. Commit the entire reservation here to prevent another
|
| 2311 | * process from stealing the pages as they are added to the pool but
|
| 2312 | * before they are reserved.
|
| 2313 |
|
| 2314 | needed += allocated;
|
| 2315 | h->resv_huge_pages += delta;
|
| 2316 | ret = 0;
|
| 2317 |
|
| 2318 |
|
| 2319 | list_for_each_entry_safe(folio, tmp, &surplus_list, lru) {
|
| 2320 | if ((--needed) < 0)
|
| 2321 | break;
|
| 2322 |
|
| 2323 | enqueue_hugetlb_folio(h, folio);
|
| 2324 | }
|
| 2325 | free:
|
| 2326 | spin_unlock_irq(&hugetlb_lock);
|
| 2327 |
|
| 2328 | |
| 2329 | * Free unnecessary surplus pages to the buddy allocator.
|
| 2330 | * Pages have no ref count, call free_huge_folio directly.
|
| 2331 |
|
| 2332 | list_for_each_entry_safe(folio, tmp, &surplus_list, lru)
|
| 2333 | free_huge_folio(folio);
|
| 2334 | spin_lock_irq(&hugetlb_lock);
|
| 2335 |
|
| 2336 | return ret;
|
| 2337 | }
|
| 2338 |
|
| 2339 | |
| 2340 | * This routine has two main purposes:
|
| 2341 | * 1) Decrement the reservation count (resv_huge_pages) by the value passed
|
| 2342 | * in unused_resv_pages. This corresponds to the prior adjustments made
|
| 2343 | * to the associated reservation map.
|
| 2344 | * 2) Free any unused surplus pages that may have been allocated to satisfy
|
| 2345 | * the reservation. As many as unused_resv_pages may be freed.
|
| 2346 |
|
| 2347 | static void return_unused_surplus_pages(struct hstate *h,
|
| 2348 | unsigned long unused_resv_pages)
|
| 2349 | {
|
| 2350 | unsigned long nr_pages;
|
| 2351 | LIST_HEAD(page_list);
|
| 2352 |
|
| 2353 | lockdep_assert_held(&hugetlb_lock);
|
| 2354 |
|
| 2355 | h->resv_huge_pages -= unused_resv_pages;
|
| 2356 |
|
| 2357 | if (hstate_is_gigantic_no_runtime(h))
|
| 2358 | goto out;
|
| 2359 |
|
| 2360 | |
| 2361 | * Part (or even all) of the reservation could have been backed
|
| 2362 | * by pre-allocated pages. Only free surplus pages.
|
| 2363 |
|
| 2364 | nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
|
| 2365 |
|
| 2366 | |
| 2367 | * We want to release as many surplus pages as possible, spread
|
| 2368 | * evenly across all nodes with memory. Iterate across these nodes
|
| 2369 | * until we can no longer free unreserved surplus pages. This occurs
|
| 2370 | * when the nodes with surplus pages have no free pages.
|
| 2371 | * remove_pool_hugetlb_folio() will balance the freed pages across the
|
| 2372 | * on-line nodes with memory and will handle the hstate accounting.
|
| 2373 |
|
| 2374 | while (nr_pages--) {
|
| 2375 | struct folio *folio;
|
| 2376 |
|
| 2377 | folio = remove_pool_hugetlb_folio(h, &node_states[N_MEMORY], 1);
|
| 2378 | if (!folio)
|
| 2379 | goto out;
|
| 2380 |
|
| 2381 | list_add(&folio->lru, &page_list);
|
| 2382 | }
|
| 2383 |
|
| 2384 | out:
|
| 2385 | spin_unlock_irq(&hugetlb_lock);
|
| 2386 | update_and_free_pages_bulk(h, &page_list);
|
| 2387 | spin_lock_irq(&hugetlb_lock);
|
| 2388 | }
|
| 2389 |
|
| 2390 |
|
| 2391 | |
| 2392 | * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
|
| 2393 | * are used by the huge page allocation routines to manage reservations.
|
| 2394 | *
|
| 2395 | * vma_needs_reservation is called to determine if the huge page at addr
|
| 2396 | * within the vma has an associated reservation. If a reservation is
|
| 2397 | * needed, the value 1 is returned. The caller is then responsible for
|
| 2398 | * managing the global reservation and subpool usage counts. After
|
| 2399 | * the huge page has been allocated, vma_commit_reservation is called
|
| 2400 | * to add the page to the reservation map. If the page allocation fails,
|
| 2401 | * the reservation must be ended instead of committed. vma_end_reservation
|
| 2402 | * is called in such cases.
|
| 2403 | *
|
| 2404 | * In the normal case, vma_commit_reservation returns the same value
|
| 2405 | * as the preceding vma_needs_reservation call. The only time this
|
| 2406 | * is not the case is if a reserve map was changed between calls. It
|
| 2407 | * is the responsibility of the caller to notice the difference and
|
| 2408 | * take appropriate action.
|
| 2409 | *
|
| 2410 | * vma_add_reservation is used in error paths where a reservation must
|
| 2411 | * be restored when a newly allocated huge page must be freed. It is
|
| 2412 | * to be called after calling vma_needs_reservation to determine if a
|
| 2413 | * reservation exists.
|
| 2414 | *
|
| 2415 | * vma_del_reservation is used in error paths where an entry in the reserve
|
| 2416 | * map was created during huge page allocation and must be removed. It is to
|
| 2417 | * be called after calling vma_needs_reservation to determine if a reservation
|
| 2418 | * exists.
|
| 2419 |
|
| 2420 | enum vma_resv_mode {
|
| 2421 | VMA_NEEDS_RESV,
|
| 2422 | VMA_COMMIT_RESV,
|
| 2423 | VMA_END_RESV,
|
| 2424 | VMA_ADD_RESV,
|
| 2425 | VMA_DEL_RESV,
|
| 2426 | };
|
| 2427 | static long __vma_reservation_common(struct hstate *h,
|
| 2428 | struct vm_area_struct *vma, unsigned long addr,
|
| 2429 | enum vma_resv_mode mode)
|
| 2430 | {
|
| 2431 | struct resv_map *resv;
|
| 2432 | pgoff_t idx;
|
| 2433 | long ret;
|
| 2434 | long dummy_out_regions_needed;
|
| 2435 |
|
| 2436 | resv = vma_resv_map(vma);
|
| 2437 | if (!resv)
|
| 2438 | return 1;
|
| 2439 |
|
| 2440 | idx = vma_hugecache_offset(h, vma, addr);
|
| 2441 | switch (mode) {
|
| 2442 | case VMA_NEEDS_RESV:
|
| 2443 | ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
|
| 2444 | |
| 2445 | * 1 page, and that adding to resv map a 1 page entry can only
|
| 2446 | * ever require 1 region.
|
| 2447 |
|
| 2448 | VM_BUG_ON(dummy_out_regions_needed != 1);
|
| 2449 | break;
|
| 2450 | case VMA_COMMIT_RESV:
|
| 2451 | ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
|
| 2452 |
|
| 2453 | VM_BUG_ON(ret < 0);
|
| 2454 | break;
|
| 2455 | case VMA_END_RESV:
|
| 2456 | region_abort(resv, idx, idx + 1, 1);
|
| 2457 | ret = 0;
|
| 2458 | break;
|
| 2459 | case VMA_ADD_RESV:
|
| 2460 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 2461 | ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
|
| 2462 |
|
| 2463 | VM_BUG_ON(ret < 0);
|
| 2464 | } else {
|
| 2465 | region_abort(resv, idx, idx + 1, 1);
|
| 2466 | ret = region_del(resv, idx, idx + 1);
|
| 2467 | }
|
| 2468 | break;
|
| 2469 | case VMA_DEL_RESV:
|
| 2470 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 2471 | region_abort(resv, idx, idx + 1, 1);
|
| 2472 | ret = region_del(resv, idx, idx + 1);
|
| 2473 | } else {
|
| 2474 | ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
|
| 2475 |
|
| 2476 | VM_BUG_ON(ret < 0);
|
| 2477 | }
|
| 2478 | break;
|
| 2479 | default:
|
| 2480 | BUG();
|
| 2481 | }
|
| 2482 |
|
| 2483 | if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
|
| 2484 | return ret;
|
| 2485 | |
| 2486 | * We know private mapping must have HPAGE_RESV_OWNER set.
|
| 2487 | *
|
| 2488 | * In most cases, reserves always exist for private mappings.
|
| 2489 | * However, a file associated with mapping could have been
|
| 2490 | * hole punched or truncated after reserves were consumed.
|
| 2491 | * As subsequent fault on such a range will not use reserves.
|
| 2492 | * Subtle - The reserve map for private mappings has the
|
| 2493 | * opposite meaning than that of shared mappings. If NO
|
| 2494 | * entry is in the reserve map, it means a reservation exists.
|
| 2495 | * If an entry exists in the reserve map, it means the
|
| 2496 | * reservation has already been consumed. As a result, the
|
| 2497 | * return value of this routine is the opposite of the
|
| 2498 | * value returned from reserve map manipulation routines above.
|
| 2499 |
|
| 2500 | if (ret > 0)
|
| 2501 | return 0;
|
| 2502 | if (ret == 0)
|
| 2503 | return 1;
|
| 2504 | return ret;
|
| 2505 | }
|
| 2506 |
|
| 2507 | static long vma_needs_reservation(struct hstate *h,
|
| 2508 | struct vm_area_struct *vma, unsigned long addr)
|
| 2509 | {
|
| 2510 | return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
|
| 2511 | }
|
| 2512 |
|
| 2513 | static long vma_commit_reservation(struct hstate *h,
|
| 2514 | struct vm_area_struct *vma, unsigned long addr)
|
| 2515 | {
|
| 2516 | return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
|
| 2517 | }
|
| 2518 |
|
| 2519 | static void vma_end_reservation(struct hstate *h,
|
| 2520 | struct vm_area_struct *vma, unsigned long addr)
|
| 2521 | {
|
| 2522 | (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
|
| 2523 | }
|
| 2524 |
|
| 2525 | static long vma_add_reservation(struct hstate *h,
|
| 2526 | struct vm_area_struct *vma, unsigned long addr)
|
| 2527 | {
|
| 2528 | return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
|
| 2529 | }
|
| 2530 |
|
| 2531 | static long vma_del_reservation(struct hstate *h,
|
| 2532 | struct vm_area_struct *vma, unsigned long addr)
|
| 2533 | {
|
| 2534 | return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
|
| 2535 | }
|
| 2536 |
|
| 2537 | |
| 2538 | * This routine is called to restore reservation information on error paths.
|
| 2539 | * It should ONLY be called for folios allocated via alloc_hugetlb_folio(),
|
| 2540 | * and the hugetlb mutex should remain held when calling this routine.
|
| 2541 | *
|
| 2542 | * It handles two specific cases:
|
| 2543 | * 1) A reservation was in place and the folio consumed the reservation.
|
| 2544 | * hugetlb_restore_reserve is set in the folio.
|
| 2545 | * 2) No reservation was in place for the page, so hugetlb_restore_reserve is
|
| 2546 | * not set. However, alloc_hugetlb_folio always updates the reserve map.
|
| 2547 | *
|
| 2548 | * In case 1, free_huge_folio later in the error path will increment the
|
| 2549 | * global reserve count. But, free_huge_folio does not have enough context
|
| 2550 | * to adjust the reservation map. This case deals primarily with private
|
| 2551 | * mappings. Adjust the reserve map here to be consistent with global
|
| 2552 | * reserve count adjustments to be made by free_huge_folio. Make sure the
|
| 2553 | * reserve map indicates there is a reservation present.
|
| 2554 | *
|
| 2555 | * In case 2, simply undo reserve map modifications done by alloc_hugetlb_folio.
|
| 2556 |
|
| 2557 | void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
|
| 2558 | unsigned long address, struct folio *folio)
|
| 2559 | {
|
| 2560 | long rc = vma_needs_reservation(h, vma, address);
|
| 2561 |
|
| 2562 | if (folio_test_hugetlb_restore_reserve(folio)) {
|
| 2563 | if (unlikely(rc < 0))
|
| 2564 | |
| 2565 | * Rare out of memory condition in reserve map
|
| 2566 | * manipulation. Clear hugetlb_restore_reserve so
|
| 2567 | * that global reserve count will not be incremented
|
| 2568 | * by free_huge_folio. This will make it appear
|
| 2569 | * as though the reservation for this folio was
|
| 2570 | * consumed. This may prevent the task from
|
| 2571 | * faulting in the folio at a later time. This
|
| 2572 | * is better than inconsistent global huge page
|
| 2573 | * accounting of reserve counts.
|
| 2574 |
|
| 2575 | folio_clear_hugetlb_restore_reserve(folio);
|
| 2576 | else if (rc)
|
| 2577 | (void)vma_add_reservation(h, vma, address);
|
| 2578 | else
|
| 2579 | vma_end_reservation(h, vma, address);
|
| 2580 | } else {
|
| 2581 | if (!rc) {
|
| 2582 | |
| 2583 | * This indicates there is an entry in the reserve map
|
| 2584 | * not added by alloc_hugetlb_folio. We know it was added
|
| 2585 | * before the alloc_hugetlb_folio call, otherwise
|
| 2586 | * hugetlb_restore_reserve would be set on the folio.
|
| 2587 | * Remove the entry so that a subsequent allocation
|
| 2588 | * does not consume a reservation.
|
| 2589 |
|
| 2590 | rc = vma_del_reservation(h, vma, address);
|
| 2591 | if (rc < 0)
|
| 2592 | |
| 2593 | * VERY rare out of memory condition. Since
|
| 2594 | * we can not delete the entry, set
|
| 2595 | * hugetlb_restore_reserve so that the reserve
|
| 2596 | * count will be incremented when the folio
|
| 2597 | * is freed. This reserve will be consumed
|
| 2598 | * on a subsequent allocation.
|
| 2599 |
|
| 2600 | folio_set_hugetlb_restore_reserve(folio);
|
| 2601 | } else if (rc < 0) {
|
| 2602 | |
| 2603 | * Rare out of memory condition from
|
| 2604 | * vma_needs_reservation call. Memory allocation is
|
| 2605 | * only attempted if a new entry is needed. Therefore,
|
| 2606 | * this implies there is not an entry in the
|
| 2607 | * reserve map.
|
| 2608 | *
|
| 2609 | * For shared mappings, no entry in the map indicates
|
| 2610 | * no reservation. We are done.
|
| 2611 |
|
| 2612 | if (!(vma->vm_flags & VM_MAYSHARE))
|
| 2613 | |
| 2614 | * For private mappings, no entry indicates
|
| 2615 | * a reservation is present. Since we can
|
| 2616 | * not add an entry, set hugetlb_restore_reserve
|
| 2617 | * on the folio so reserve count will be
|
| 2618 | * incremented when freed. This reserve will
|
| 2619 | * be consumed on a subsequent allocation.
|
| 2620 |
|
| 2621 | folio_set_hugetlb_restore_reserve(folio);
|
| 2622 | } else {
|
| 2623 | |
| 2624 | * No reservation present, do nothing
|
| 2625 |
|
| 2626 | vma_end_reservation(h, vma, address);
|
| 2627 | }
|
| 2628 | }
|
| 2629 | }
|
| 2630 |
|
| 2631 | |
| 2632 | * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
|
| 2633 | * the old one
|
| 2634 | * @old_folio: Old folio to dissolve
|
| 2635 | * @list: List to isolate the page in case we need to
|
| 2636 | * Returns 0 on success, otherwise negated error.
|
| 2637 |
|
| 2638 | static int alloc_and_dissolve_hugetlb_folio(struct folio *old_folio,
|
| 2639 | struct list_head *list)
|
| 2640 | {
|
| 2641 | gfp_t gfp_mask;
|
| 2642 | struct hstate *h;
|
| 2643 | int nid = folio_nid(old_folio);
|
| 2644 | struct folio *new_folio = NULL;
|
| 2645 | int ret = 0;
|
| 2646 |
|
| 2647 | retry:
|
| 2648 | |
| 2649 | * The old_folio might have been dissolved from under our feet, so make sure
|
| 2650 | * to carefully check the state under the lock.
|
| 2651 |
|
| 2652 | spin_lock_irq(&hugetlb_lock);
|
| 2653 | if (!folio_test_hugetlb(old_folio)) {
|
| 2654 | |
| 2655 | * Freed from under us. Drop new_folio too.
|
| 2656 |
|
| 2657 | goto free_new;
|
| 2658 | } else if (folio_ref_count(old_folio)) {
|
| 2659 | bool isolated;
|
| 2660 |
|
| 2661 | |
| 2662 | * Someone has grabbed the folio, try to isolate it here.
|
| 2663 | * Fail with -EBUSY if not possible.
|
| 2664 |
|
| 2665 | spin_unlock_irq(&hugetlb_lock);
|
| 2666 | isolated = folio_isolate_hugetlb(old_folio, list);
|
| 2667 | ret = isolated ? 0 : -EBUSY;
|
| 2668 | spin_lock_irq(&hugetlb_lock);
|
| 2669 | goto free_new;
|
| 2670 | } else if (!folio_test_hugetlb_freed(old_folio)) {
|
| 2671 | |
| 2672 | * Folio's refcount is 0 but it has not been enqueued in the
|
| 2673 | * freelist yet. Race window is small, so we can succeed here if
|
| 2674 | * we retry.
|
| 2675 |
|
| 2676 | spin_unlock_irq(&hugetlb_lock);
|
| 2677 | cond_resched();
|
| 2678 | goto retry;
|
| 2679 | } else {
|
| 2680 | h = folio_hstate(old_folio);
|
| 2681 | if (!new_folio) {
|
| 2682 | spin_unlock_irq(&hugetlb_lock);
|
| 2683 | gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
|
| 2684 | new_folio = alloc_fresh_hugetlb_folio(h, gfp_mask,
|
| 2685 | nid, NULL);
|
| 2686 | if (!new_folio)
|
| 2687 | return -ENOMEM;
|
| 2688 | goto retry;
|
| 2689 | }
|
| 2690 |
|
| 2691 | |
| 2692 | * Ok, old_folio is still a genuine free hugepage. Remove it from
|
| 2693 | * the freelist and decrease the counters. These will be
|
| 2694 | * incremented again when calling account_new_hugetlb_folio()
|
| 2695 | * and enqueue_hugetlb_folio() for new_folio. The counters will
|
| 2696 | * remain stable since this happens under the lock.
|
| 2697 |
|
| 2698 | remove_hugetlb_folio(h, old_folio, false);
|
| 2699 |
|
| 2700 | |
| 2701 | * Ref count on new_folio is already zero as it was dropped
|
| 2702 | * earlier. It can be directly added to the pool free list.
|
| 2703 |
|
| 2704 | account_new_hugetlb_folio(h, new_folio);
|
| 2705 | enqueue_hugetlb_folio(h, new_folio);
|
| 2706 |
|
| 2707 | |
| 2708 | * Folio has been replaced, we can safely free the old one.
|
| 2709 |
|
| 2710 | spin_unlock_irq(&hugetlb_lock);
|
| 2711 | update_and_free_hugetlb_folio(h, old_folio, false);
|
| 2712 | }
|
| 2713 |
|
| 2714 | return ret;
|
| 2715 |
|
| 2716 | free_new:
|
| 2717 | spin_unlock_irq(&hugetlb_lock);
|
| 2718 | if (new_folio)
|
| 2719 | update_and_free_hugetlb_folio(h, new_folio, false);
|
| 2720 |
|
| 2721 | return ret;
|
| 2722 | }
|
| 2723 |
|
| 2724 | int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list)
|
| 2725 | {
|
| 2726 | int ret = -EBUSY;
|
| 2727 |
|
| 2728 |
|
| 2729 | if (!folio_test_hugetlb(folio))
|
| 2730 | return 0;
|
| 2731 |
|
| 2732 | |
| 2733 | * Fence off gigantic pages as there is a cyclic dependency between
|
| 2734 | * alloc_contig_range and them. Return -ENOMEM as this has the effect
|
| 2735 | * of bailing out right away without further retrying.
|
| 2736 |
|
| 2737 | if (order_is_gigantic(folio_order(folio)))
|
| 2738 | return -ENOMEM;
|
| 2739 |
|
| 2740 | if (folio_ref_count(folio) && folio_isolate_hugetlb(folio, list))
|
| 2741 | ret = 0;
|
| 2742 | else if (!folio_ref_count(folio))
|
| 2743 | ret = alloc_and_dissolve_hugetlb_folio(folio, list);
|
| 2744 |
|
| 2745 | return ret;
|
| 2746 | }
|
| 2747 |
|
| 2748 | |
| 2749 | * replace_free_hugepage_folios - Replace free hugepage folios in a given pfn
|
| 2750 | * range with new folios.
|
| 2751 | * @start_pfn: start pfn of the given pfn range
|
| 2752 | * @end_pfn: end pfn of the given pfn range
|
| 2753 | * Returns 0 on success, otherwise negated error.
|
| 2754 |
|
| 2755 | int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn)
|
| 2756 | {
|
| 2757 | unsigned long nr = 0;
|
| 2758 | struct page *page;
|
| 2759 | struct hstate *h;
|
| 2760 | LIST_HEAD(list);
|
| 2761 | int ret = 0;
|
| 2762 |
|
| 2763 |
|
| 2764 | for_each_hstate(h) {
|
| 2765 | if (hstate_is_gigantic(h))
|
| 2766 | continue;
|
| 2767 |
|
| 2768 | nr += h->free_huge_pages;
|
| 2769 | if (nr)
|
| 2770 | break;
|
| 2771 | }
|
| 2772 |
|
| 2773 | if (!nr)
|
| 2774 | return 0;
|
| 2775 |
|
| 2776 | while (start_pfn < end_pfn) {
|
| 2777 | page = pfn_to_page(start_pfn);
|
| 2778 | nr = 1;
|
| 2779 |
|
| 2780 | if (PageHuge(page) || PageCompound(page)) {
|
| 2781 | struct folio *folio = page_folio(page);
|
| 2782 |
|
| 2783 | nr = folio_nr_pages(folio) - folio_page_idx(folio, page);
|
| 2784 |
|
| 2785 | |
| 2786 | * Don't disrupt normal path by vainly holding
|
| 2787 | * hugetlb_lock
|
| 2788 |
|
| 2789 | if (folio_test_hugetlb(folio) && !folio_ref_count(folio)) {
|
| 2790 | if (order_is_gigantic(folio_order(folio))) {
|
| 2791 | ret = -ENOMEM;
|
| 2792 | break;
|
| 2793 | }
|
| 2794 |
|
| 2795 | ret = alloc_and_dissolve_hugetlb_folio(folio, &list);
|
| 2796 | if (ret)
|
| 2797 | break;
|
| 2798 |
|
| 2799 | putback_movable_pages(&list);
|
| 2800 | }
|
| 2801 | } else if (PageBuddy(page)) {
|
| 2802 | |
| 2803 | * Buddy order check without zone lock is unsafe and
|
| 2804 | * the order is maybe invalid, but race should be
|
| 2805 | * small, and the worst thing is skipping free hugetlb.
|
| 2806 |
|
| 2807 | const unsigned int order = buddy_order_unsafe(page);
|
| 2808 |
|
| 2809 | if (order <= MAX_PAGE_ORDER)
|
| 2810 | nr = 1UL << order;
|
| 2811 | }
|
| 2812 | start_pfn += nr;
|
| 2813 | }
|
| 2814 |
|
| 2815 | return ret;
|
| 2816 | }
|
| 2817 |
|
| 2818 | void wait_for_freed_hugetlb_folios(void)
|
| 2819 | {
|
| 2820 | if (llist_empty(&hpage_freelist))
|
| 2821 | return;
|
| 2822 |
|
| 2823 | flush_work(&free_hpage_work);
|
| 2824 | }
|
| 2825 |
|
| 2826 | |
| 2827 | * hugetlb_alloc_folio - Allocate a hugetlb folio.
|
| 2828 | * @h: Hugetlb state control block.
|
| 2829 | * @mpoli: Interpreted memory policy to use for allocation.
|
| 2830 | * @alloc_flags: Flags controlling the allocation behavior.
|
| 2831 | *
|
| 2832 | * Allocates a hugetlb folio and handles cgroup charging and global hstate
|
| 2833 | * reservations.
|
| 2834 | *
|
| 2835 | * Return: A pointer to the allocated folio, or an ERR_PTR on failure.
|
| 2836 | * -ENOSPC if cgroup charging fails or no folio is available.
|
| 2837 | * -ENOMEM if mem cgroup charging fails.
|
| 2838 |
|
| 2839 | struct folio *hugetlb_alloc_folio(struct hstate *h,
|
| 2840 | struct mempolicy_interpreted *mpoli, u8 alloc_flags)
|
| 2841 | {
|
| 2842 | bool charge_hugetlb_cgroup_rsvd = alloc_flags &
|
| 2843 | HUGETLB_ALLOC_CHARG_CGROUP_RSVD;
|
| 2844 | bool use_global_reservation = alloc_flags &
|
| 2845 | HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS;
|
| 2846 | size_t nr_pages = pages_per_huge_page(h);
|
| 2847 | struct hugetlb_cgroup *h_cg_rsvd = NULL;
|
| 2848 | struct hugetlb_cgroup *h_cg = NULL;
|
| 2849 | gfp_t gfp = htlb_alloc_mask(h);
|
| 2850 | int idx = hstate_index(h);
|
| 2851 | struct folio *folio;
|
| 2852 | int ret;
|
| 2853 |
|
| 2854 | if (charge_hugetlb_cgroup_rsvd &&
|
| 2855 | hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd))
|
| 2856 | return ERR_PTR(-ENOSPC);
|
| 2857 |
|
| 2858 | if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) {
|
| 2859 | ret = -ENOSPC;
|
| 2860 | goto err_uncharge_hugetlb_cgroup_rsvd;
|
| 2861 | }
|
| 2862 |
|
| 2863 | spin_lock_irq(&hugetlb_lock);
|
| 2864 |
|
| 2865 | folio = NULL;
|
| 2866 | if (use_global_reservation || available_huge_pages(h))
|
| 2867 | folio = dequeue_hugetlb_folio(h, gfp, mpoli);
|
| 2868 |
|
| 2869 | if (!folio) {
|
| 2870 | spin_unlock_irq(&hugetlb_lock);
|
| 2871 | folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli);
|
| 2872 | if (!folio) {
|
| 2873 | ret = -ENOSPC;
|
| 2874 | goto err_uncharge_hugetlb_cgroup;
|
| 2875 | }
|
| 2876 | spin_lock_irq(&hugetlb_lock);
|
| 2877 | list_add(&folio->lru, &h->hugepage_activelist);
|
| 2878 | folio_ref_unfreeze(folio, 1);
|
| 2879 | }
|
| 2880 |
|
| 2881 | if (use_global_reservation) {
|
| 2882 | folio_set_hugetlb_restore_reserve(folio);
|
| 2883 | h->resv_huge_pages--;
|
| 2884 | }
|
| 2885 |
|
| 2886 | hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio);
|
| 2887 |
|
| 2888 | if (charge_hugetlb_cgroup_rsvd) {
|
| 2889 | hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd,
|
| 2890 | folio);
|
| 2891 | }
|
| 2892 |
|
| 2893 | spin_unlock_irq(&hugetlb_lock);
|
| 2894 |
|
| 2895 | ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL);
|
| 2896 | |
| 2897 | * Unconditionally increment NR_HUGETLB here because if
|
| 2898 | * mem_cgroup_charge_hugetlb failed, freeing the page will
|
| 2899 | * decrement NR_HUGETLB.
|
| 2900 |
|
| 2901 | lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages);
|
| 2902 |
|
| 2903 | if (ret == -ENOMEM) {
|
| 2904 | free_huge_folio(folio);
|
| 2905 | |
| 2906 | * Skip uncharging hugetlb_cgroup since the charges
|
| 2907 | * were committed to the folio and freeing the folio
|
| 2908 | * would have cleared those up.
|
| 2909 |
|
| 2910 | return ERR_PTR(ret);
|
| 2911 | }
|
| 2912 |
|
| 2913 | return folio;
|
| 2914 |
|
| 2915 | err_uncharge_hugetlb_cgroup:
|
| 2916 | hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg);
|
| 2917 | err_uncharge_hugetlb_cgroup_rsvd:
|
| 2918 | if (charge_hugetlb_cgroup_rsvd)
|
| 2919 | hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd);
|
| 2920 |
|
| 2921 | return ERR_PTR(ret);
|
| 2922 | }
|
| 2923 |
|
| 2924 | typedef enum {
|
| 2925 | |
| 2926 | * For either 0/1: we checked the per-vma resv map, and one resv
|
| 2927 | * count either can be reused (0), or an extra needed (1).
|
| 2928 |
|
| 2929 | MAP_CHG_REUSE = 0,
|
| 2930 | MAP_CHG_NEEDED = 1,
|
| 2931 | |
| 2932 | * Cannot use per-vma resv count can be used, hence a new resv
|
| 2933 | * count is enforced.
|
| 2934 | *
|
| 2935 | * NOTE: This is mostly identical to MAP_CHG_NEEDED, except
|
| 2936 | * that currently vma_needs_reservation() has an unwanted side
|
| 2937 | * effect to either use end() or commit() to complete the
|
| 2938 | * transaction. Hence it needs to differentiate from NEEDED.
|
| 2939 |
|
| 2940 | MAP_CHG_ENFORCED = 2,
|
| 2941 | } map_chg_state;
|
| 2942 |
|
| 2943 | |
| 2944 | * NOTE! "cow_from_owner" represents a very hacky usage only used in CoW
|
| 2945 | * faults of hugetlb private mappings on top of a non-page-cache folio (in
|
| 2946 | * which case even if there's a private vma resv map it won't cover such
|
| 2947 | * allocation). New call sites should (probably) never set it to true!!
|
| 2948 | * When it's set, the allocation will bypass all vma level reservations.
|
| 2949 |
|
| 2950 | struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
|
| 2951 | unsigned long addr, bool cow_from_owner)
|
| 2952 | {
|
| 2953 | struct hugepage_subpool *spool = subpool_vma(vma);
|
| 2954 | struct hstate *h = hstate_vma(vma);
|
| 2955 | struct folio *folio;
|
| 2956 | long retval, gbl_chg, gbl_reserve;
|
| 2957 | map_chg_state map_chg;
|
| 2958 | struct mempolicy_interpreted mpoli;
|
| 2959 | gfp_t gfp = htlb_alloc_mask(h);
|
| 2960 | struct mempolicy *mpol;
|
| 2961 | nodemask_t *nodemask;
|
| 2962 | u8 alloc_flags = 0;
|
| 2963 | int nid;
|
| 2964 | int ret;
|
| 2965 |
|
| 2966 |
|
| 2967 | if (cow_from_owner) {
|
| 2968 | |
| 2969 | * Special case! Since it's a CoW on top of a reserved
|
| 2970 | * page, the private resv map doesn't count. So it cannot
|
| 2971 | * consume the per-vma resv map even if it's reserved.
|
| 2972 |
|
| 2973 | map_chg = MAP_CHG_ENFORCED;
|
| 2974 | } else {
|
| 2975 | |
| 2976 | * Examine the region/reserve map to determine if the process
|
| 2977 | * has a reservation for the page to be allocated. A return
|
| 2978 | * code of zero indicates a reservation exists (no change).
|
| 2979 |
|
| 2980 | retval = vma_needs_reservation(h, vma, addr);
|
| 2981 | if (retval < 0)
|
| 2982 | return ERR_PTR(-ENOMEM);
|
| 2983 | map_chg = retval ? MAP_CHG_NEEDED : MAP_CHG_REUSE;
|
| 2984 | }
|
| 2985 |
|
| 2986 | |
| 2987 | * Whether we need a separate global reservation?
|
| 2988 | *
|
| 2989 | * Processes that did not create the mapping will have no
|
| 2990 | * reserves as indicated by the region/reserve map. Check
|
| 2991 | * that the allocation will not exceed the subpool limit.
|
| 2992 | * Or if it can get one from the pool reservation directly.
|
| 2993 |
|
| 2994 | if (map_chg) {
|
| 2995 | gbl_chg = hugepage_subpool_get_pages(spool, 1);
|
| 2996 | if (gbl_chg < 0) {
|
| 2997 | ret = -ENOSPC;
|
| 2998 | goto out_end_reservation;
|
| 2999 | }
|
| 3000 | } else {
|
| 3001 | |
| 3002 | * If we have the vma reservation ready, no need for extra
|
| 3003 | * global reservation.
|
| 3004 |
|
| 3005 | gbl_chg = 0;
|
| 3006 | }
|
| 3007 |
|
| 3008 | |
| 3009 | * If allocation doesn't reuse a reservation in the resv_map,
|
| 3010 | * charge for the reservation.
|
| 3011 |
|
| 3012 | if (map_chg != MAP_CHG_REUSE)
|
| 3013 | alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD;
|
| 3014 |
|
| 3015 | |
| 3016 | * gbl_chg == 0 indicates a reservation exists for this
|
| 3017 | * allocation, so try to use it.
|
| 3018 |
|
| 3019 | if (gbl_chg == 0)
|
| 3020 | alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS;
|
| 3021 |
|
| 3022 |
|
| 3023 | nid = huge_node(vma, addr, gfp, &mpol, &nodemask);
|
| 3024 | mpoli = (struct mempolicy_interpreted){
|
| 3025 | .nid = nid,
|
| 3026 | #ifdef CONFIG_NUMA
|
| 3027 | .mode = mpol ? mpol->mode : MPOL_DEFAULT,
|
| 3028 | #else
|
| 3029 | .mode = MPOL_DEFAULT,
|
| 3030 | #endif
|
| 3031 | .nodemask = nodemask,
|
| 3032 | };
|
| 3033 |
|
| 3034 | folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags);
|
| 3035 |
|
| 3036 | mpol_cond_put(mpol);
|
| 3037 |
|
| 3038 | if (IS_ERR(folio)) {
|
| 3039 | ret = PTR_ERR(folio);
|
| 3040 | goto out_subpool_put;
|
| 3041 | }
|
| 3042 |
|
| 3043 | hugetlb_set_folio_subpool(folio, spool);
|
| 3044 |
|
| 3045 | if (map_chg != MAP_CHG_ENFORCED) {
|
| 3046 |
|
| 3047 | retval = vma_commit_reservation(h, vma, addr);
|
| 3048 | |
| 3049 | * Check for possible race conditions. When it happens..
|
| 3050 | * The page was added to the reservation map between
|
| 3051 | * vma_needs_reservation and vma_commit_reservation.
|
| 3052 | * This indicates a race with hugetlb_reserve_pages.
|
| 3053 | * Adjust for the subpool count incremented above AND
|
| 3054 | * in hugetlb_reserve_pages for the same page. Also,
|
| 3055 | * the reservation count added in hugetlb_reserve_pages
|
| 3056 | * no longer applies.
|
| 3057 |
|
| 3058 | if (unlikely(map_chg == MAP_CHG_NEEDED && retval == 0)) {
|
| 3059 | long rsv_adjust;
|
| 3060 |
|
| 3061 | rsv_adjust = hugepage_subpool_put_pages(spool, 1);
|
| 3062 | hugetlb_acct_memory(h, -rsv_adjust);
|
| 3063 | spin_lock_irq(&hugetlb_lock);
|
| 3064 | hugetlb_cgroup_uncharge_folio_rsvd(
|
| 3065 | hstate_index(h), pages_per_huge_page(h), folio);
|
| 3066 | spin_unlock_irq(&hugetlb_lock);
|
| 3067 | }
|
| 3068 | }
|
| 3069 |
|
| 3070 | return folio;
|
| 3071 |
|
| 3072 | out_subpool_put:
|
| 3073 | |
| 3074 | * put page to subpool iff the quota of subpool's rsv_hpages is used
|
| 3075 | * during hugepage_subpool_get_pages.
|
| 3076 |
|
| 3077 | if (map_chg && !gbl_chg) {
|
| 3078 | gbl_reserve = hugepage_subpool_put_pages(spool, 1);
|
| 3079 | hugetlb_acct_memory(h, -gbl_reserve);
|
| 3080 | }
|
| 3081 |
|
| 3082 | out_end_reservation:
|
| 3083 | if (map_chg != MAP_CHG_ENFORCED)
|
| 3084 | vma_end_reservation(h, vma, addr);
|
| 3085 | return ERR_PTR(ret);
|
| 3086 | }
|
| 3087 |
|
| 3088 | static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact)
|
| 3089 | {
|
| 3090 | if (hugetlb_early_cma(h))
|
| 3091 | return hugetlb_cma_alloc_bootmem(h, nid, node_exact);
|
| 3092 |
|
| 3093 | return memblock_alloc_hugetlb(huge_page_size(h), nid, node_exact);
|
| 3094 | }
|
| 3095 |
|
| 3096 | void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
|
| 3097 | __attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
|
| 3098 | void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid)
|
| 3099 | {
|
| 3100 | int nr_nodes, node = nid;
|
| 3101 |
|
| 3102 |
|
| 3103 | if (nid != NUMA_NO_NODE)
|
| 3104 | return alloc_bootmem(h, node, true);
|
| 3105 |
|
| 3106 |
|
| 3107 | for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node,
|
| 3108 | &hugetlb_bootmem_nodes)
|
| 3109 | return alloc_bootmem(h, node, false);
|
| 3110 |
|
| 3111 | return NULL;
|
| 3112 | }
|
| 3113 |
|
| 3114 | static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
|
| 3115 | {
|
| 3116 | unsigned long pfn;
|
| 3117 | unsigned int nid_request = nid;
|
| 3118 | struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid);
|
| 3119 |
|
| 3120 | if (!m)
|
| 3121 | return false;
|
| 3122 |
|
| 3123 | pfn = PHYS_PFN(__pa(m));
|
| 3124 | nid = early_pfn_to_nid(pfn);
|
| 3125 | |
| 3126 | * Use the beginning of the huge page to store the huge_bootmem_page
|
| 3127 | * struct (until gather_bootmem puts them into the mem_map).
|
| 3128 | *
|
| 3129 | * Put them into a private list first because mem_map is not up yet.
|
| 3130 |
|
| 3131 | INIT_LIST_HEAD(&m->list);
|
| 3132 | m->hstate = h;
|
| 3133 | m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0;
|
| 3134 |
|
| 3135 |
|
| 3136 | if (!hugetlb_early_cma(h) &&
|
| 3137 | pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) {
|
| 3138 | |
| 3139 | * If the allocated page is on a different node than requested
|
| 3140 | * (e.g., on PowerPC LPARs), put it on the requested node's list,
|
| 3141 | * because hugetlb_free_cross_zone_pages() only frees cross-zone
|
| 3142 | * pages belonging to the requested node.
|
| 3143 |
|
| 3144 | if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request))
|
| 3145 | list_add(&m->list, &huge_boot_pages[nid_request]);
|
| 3146 | else
|
| 3147 | list_add(&m->list, &huge_boot_pages[nid]);
|
| 3148 | } else {
|
| 3149 | list_add_tail(&m->list, &huge_boot_pages[nid]);
|
| 3150 | m->flags |= HUGE_BOOTMEM_ZONES_VALID;
|
| 3151 | |
| 3152 | * Only initialize the head struct page in memmap_init_reserved_pages,
|
| 3153 | * rest of the struct pages will be initialized by the HugeTLB
|
| 3154 | * subsystem itself.
|
| 3155 | * The head struct page is used to get folio information by the HugeTLB
|
| 3156 | * subsystem like zone id and node id.
|
| 3157 |
|
| 3158 | memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
|
| 3159 | huge_page_size(h) - PAGE_SIZE);
|
| 3160 | }
|
| 3161 |
|
| 3162 | return true;
|
| 3163 | }
|
| 3164 |
|
| 3165 |
|
| 3166 | static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
|
| 3167 | struct hstate *h,
|
| 3168 | unsigned long start_page_number,
|
| 3169 | unsigned long end_page_number)
|
| 3170 | {
|
| 3171 | enum zone_type zone = folio_zonenum(folio);
|
| 3172 | int nid = folio_nid(folio);
|
| 3173 | struct page *page = folio_page(folio, start_page_number);
|
| 3174 | unsigned long head_pfn = folio_pfn(folio);
|
| 3175 | unsigned long pfn, end_pfn = head_pfn + end_page_number;
|
| 3176 | unsigned int order = huge_page_order(h);
|
| 3177 |
|
| 3178 | |
| 3179 | * As we marked all tail pages with memblock_reserved_mark_noinit(),
|
| 3180 | * we must initialize them ourselves here.
|
| 3181 |
|
| 3182 | for (pfn = head_pfn + start_page_number; pfn < end_pfn; page++, pfn++) {
|
| 3183 | __init_single_page(page, pfn, zone, nid);
|
| 3184 | prep_compound_tail(page, &folio->page, order);
|
| 3185 | set_page_count(page, 0);
|
| 3186 | }
|
| 3187 | }
|
| 3188 |
|
| 3189 | static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
|
| 3190 | struct hstate *h,
|
| 3191 | unsigned long nr_pages)
|
| 3192 | {
|
| 3193 | int ret;
|
| 3194 |
|
| 3195 | |
| 3196 | * This is an open-coded prep_compound_page() whereby we avoid
|
| 3197 | * walking pages twice by initializing/preparing+freezing them in the
|
| 3198 | * same go.
|
| 3199 |
|
| 3200 | __folio_clear_reserved(folio);
|
| 3201 | __folio_set_head(folio);
|
| 3202 | ret = folio_ref_freeze(folio, 1);
|
| 3203 | VM_BUG_ON(!ret);
|
| 3204 | hugetlb_folio_init_tail_vmemmap(folio, h, 1, nr_pages);
|
| 3205 | prep_compound_head(&folio->page, huge_page_order(h));
|
| 3206 | }
|
| 3207 |
|
| 3208 | static bool __init hugetlb_bootmem_page_prehvo(struct huge_bootmem_page *m)
|
| 3209 | {
|
| 3210 | return m->flags & HUGE_BOOTMEM_HVO;
|
| 3211 | }
|
| 3212 |
|
| 3213 | static bool __init hugetlb_bootmem_page_earlycma(struct huge_bootmem_page *m)
|
| 3214 | {
|
| 3215 | return m->flags & HUGE_BOOTMEM_CMA;
|
| 3216 | }
|
| 3217 |
|
| 3218 | |
| 3219 | * memblock-allocated pageblocks might not have the migrate type set
|
| 3220 | * if marked with the 'noinit' flag. Set it to the default (MIGRATE_MOVABLE)
|
| 3221 | * here, or MIGRATE_CMA if this was a page allocated through an early CMA
|
| 3222 | * reservation.
|
| 3223 | *
|
| 3224 | * In case of vmemmap optimized folios, the tail vmemmap pages are mapped
|
| 3225 | * read-only, but that's ok - for sparse vmemmap this does not write to
|
| 3226 | * the page structure.
|
| 3227 |
|
| 3228 | static void __init hugetlb_bootmem_init_migratetype(struct folio *folio,
|
| 3229 | struct hstate *h)
|
| 3230 | {
|
| 3231 | unsigned long nr_pages = pages_per_huge_page(h), i;
|
| 3232 |
|
| 3233 | WARN_ON_ONCE(!pageblock_aligned(folio_pfn(folio)));
|
| 3234 |
|
| 3235 | for (i = 0; i < nr_pages; i += pageblock_nr_pages) {
|
| 3236 | if (folio_test_hugetlb_cma(folio))
|
| 3237 | init_cma_pageblock(folio_page(folio, i));
|
| 3238 | else
|
| 3239 | init_pageblock_migratetype(folio_page(folio, i),
|
| 3240 | MIGRATE_MOVABLE, false);
|
| 3241 | }
|
| 3242 | }
|
| 3243 |
|
| 3244 | static void __init prep_and_add_bootmem_folios(struct hstate *h,
|
| 3245 | struct list_head *folio_list)
|
| 3246 | {
|
| 3247 | unsigned long flags;
|
| 3248 | struct folio *folio, *tmp_f;
|
| 3249 |
|
| 3250 |
|
| 3251 | hugetlb_vmemmap_optimize_bootmem_folios(h, folio_list);
|
| 3252 |
|
| 3253 | list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
|
| 3254 | if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
|
| 3255 | |
| 3256 | * If HVO fails, initialize all tail struct pages
|
| 3257 | * We do not worry about potential long lock hold
|
| 3258 | * time as this is early in boot and there should
|
| 3259 | * be no contention.
|
| 3260 |
|
| 3261 | hugetlb_folio_init_tail_vmemmap(folio, h,
|
| 3262 | HUGETLB_VMEMMAP_RESERVE_PAGES,
|
| 3263 | pages_per_huge_page(h));
|
| 3264 | }
|
| 3265 | hugetlb_bootmem_init_migratetype(folio, h);
|
| 3266 |
|
| 3267 | spin_lock_irqsave(&hugetlb_lock, flags);
|
| 3268 | account_new_hugetlb_folio(h, folio);
|
| 3269 | enqueue_hugetlb_folio(h, folio);
|
| 3270 | spin_unlock_irqrestore(&hugetlb_lock, flags);
|
| 3271 | }
|
| 3272 | }
|
| 3273 |
|
| 3274 | |
| 3275 | * Put bootmem huge pages into the standard lists after mem_map is up.
|
| 3276 | * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
|
| 3277 |
|
| 3278 | static void __init gather_bootmem_prealloc_node(unsigned long nid)
|
| 3279 | {
|
| 3280 | LIST_HEAD(folio_list);
|
| 3281 | struct huge_bootmem_page *m, *tm;
|
| 3282 | struct hstate *h = NULL, *prev_h = NULL;
|
| 3283 |
|
| 3284 | list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
|
| 3285 | struct page *page = virt_to_page(m);
|
| 3286 | struct folio *folio = (void *)page;
|
| 3287 |
|
| 3288 | h = m->hstate;
|
| 3289 | |
| 3290 | * It is possible to have multiple huge page sizes (hstates)
|
| 3291 | * in this list. If so, process each size separately.
|
| 3292 |
|
| 3293 | if (h != prev_h && prev_h != NULL)
|
| 3294 | prep_and_add_bootmem_folios(prev_h, &folio_list);
|
| 3295 | prev_h = h;
|
| 3296 |
|
| 3297 | VM_BUG_ON(!hstate_is_gigantic(h));
|
| 3298 | WARN_ON(folio_ref_count(folio) != 1);
|
| 3299 |
|
| 3300 | hugetlb_folio_init_vmemmap(folio, h,
|
| 3301 | HUGETLB_VMEMMAP_RESERVE_PAGES);
|
| 3302 | init_new_hugetlb_folio(folio);
|
| 3303 |
|
| 3304 | if (hugetlb_bootmem_page_prehvo(m))
|
| 3305 | |
| 3306 | * If pre-HVO was done, just set the
|
| 3307 | * flag, the HVO code will then skip
|
| 3308 | * this folio.
|
| 3309 |
|
| 3310 | folio_set_hugetlb_vmemmap_optimized(folio);
|
| 3311 |
|
| 3312 | if (hugetlb_bootmem_page_earlycma(m))
|
| 3313 | folio_set_hugetlb_cma(folio);
|
| 3314 |
|
| 3315 | list_add(&folio->lru, &folio_list);
|
| 3316 |
|
| 3317 | |
| 3318 | * We need to restore the 'stolen' pages to totalram_pages
|
| 3319 | * in order to fix confusing memory reports from free(1) and
|
| 3320 | * other side-effects, like CommitLimit going negative.
|
| 3321 | *
|
| 3322 | * For CMA pages, this is done in init_cma_pageblock
|
| 3323 | * (via hugetlb_bootmem_init_migratetype), so skip it here.
|
| 3324 |
|
| 3325 | if (!folio_test_hugetlb_cma(folio))
|
| 3326 | adjust_managed_page_count(page, pages_per_huge_page(h));
|
| 3327 | cond_resched();
|
| 3328 | }
|
| 3329 |
|
| 3330 | prep_and_add_bootmem_folios(h, &folio_list);
|
| 3331 | }
|
| 3332 |
|
| 3333 | static void __init gather_bootmem_prealloc_parallel(unsigned long start,
|
| 3334 | unsigned long end, void *arg)
|
| 3335 | {
|
| 3336 | int nid;
|
| 3337 |
|
| 3338 | for (nid = start; nid < end; nid++)
|
| 3339 | gather_bootmem_prealloc_node(nid);
|
| 3340 | }
|
| 3341 |
|
| 3342 | void __init hugetlb_bootmem_struct_page_init(void)
|
| 3343 | {
|
| 3344 | struct padata_mt_job job = {
|
| 3345 | .thread_fn = gather_bootmem_prealloc_parallel,
|
| 3346 | .fn_arg = NULL,
|
| 3347 | .start = 0,
|
| 3348 | .size = nr_node_ids,
|
| 3349 | .align = 1,
|
| 3350 | .min_chunk = 1,
|
| 3351 | .max_threads = num_node_state(N_MEMORY),
|
| 3352 | .numa_aware = true,
|
| 3353 | };
|
| 3354 | #ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
|
| 3355 | struct zone *zone;
|
| 3356 |
|
| 3357 | for_each_zone(zone) {
|
| 3358 | for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
|
| 3359 | struct page *tail, *p;
|
| 3360 | unsigned int order;
|
| 3361 |
|
| 3362 | tail = zone->vmemmap_tails[i];
|
| 3363 | if (!tail)
|
| 3364 | continue;
|
| 3365 |
|
| 3366 | order = i + VMEMMAP_TAIL_MIN_ORDER;
|
| 3367 | p = page_to_virt(tail);
|
| 3368 | |
| 3369 | * prep_and_add_bootmem_folios() can access pageblock
|
| 3370 | * flags on bootmem HugeTLB pages, so initialize the
|
| 3371 | * shared tail struct pages here before bootmem folios
|
| 3372 | * start using them.
|
| 3373 |
|
| 3374 | for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
|
| 3375 | init_compound_tail(p + j, NULL, order, zone);
|
| 3376 | }
|
| 3377 | }
|
| 3378 | #endif
|
| 3379 |
|
| 3380 | padata_do_multithreaded(&job);
|
| 3381 | }
|
| 3382 |
|
| 3383 | static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid)
|
| 3384 | {
|
| 3385 | unsigned long freed = 0;
|
| 3386 | struct huge_bootmem_page *m, *tmp;
|
| 3387 |
|
| 3388 | if (!hstate_is_gigantic(h))
|
| 3389 | return freed;
|
| 3390 |
|
| 3391 | list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) {
|
| 3392 | if (m->flags & HUGE_BOOTMEM_ZONES_VALID)
|
| 3393 | break;
|
| 3394 |
|
| 3395 | list_del(&m->list);
|
| 3396 | memblock_free(m, huge_page_size(h));
|
| 3397 | freed++;
|
| 3398 | }
|
| 3399 |
|
| 3400 | if (freed) {
|
| 3401 | char buf[32];
|
| 3402 |
|
| 3403 | string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf));
|
| 3404 | pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n",
|
| 3405 | freed, buf, nid);
|
| 3406 | }
|
| 3407 |
|
| 3408 | return freed;
|
| 3409 | }
|
| 3410 |
|
| 3411 | static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
|
| 3412 | {
|
| 3413 | unsigned long i;
|
| 3414 | char buf[32];
|
| 3415 | LIST_HEAD(folio_list);
|
| 3416 |
|
| 3417 | for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
|
| 3418 | if (hstate_is_gigantic(h)) {
|
| 3419 | if (!alloc_bootmem_huge_page(h, nid))
|
| 3420 | break;
|
| 3421 | } else {
|
| 3422 | struct folio *folio;
|
| 3423 | gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
|
| 3424 |
|
| 3425 | folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
|
| 3426 | &node_states[N_MEMORY], NULL);
|
| 3427 | if (!folio && !list_empty(&folio_list) &&
|
| 3428 | hugetlb_vmemmap_optimizable_size(h)) {
|
| 3429 | prep_and_add_allocated_folios(h, &folio_list);
|
| 3430 | INIT_LIST_HEAD(&folio_list);
|
| 3431 | folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
|
| 3432 | &node_states[N_MEMORY], NULL);
|
| 3433 | }
|
| 3434 | if (!folio)
|
| 3435 | break;
|
| 3436 | list_add(&folio->lru, &folio_list);
|
| 3437 | }
|
| 3438 | cond_resched();
|
| 3439 | }
|
| 3440 |
|
| 3441 | i -= hugetlb_free_cross_zone_pages(h, nid);
|
| 3442 |
|
| 3443 | if (!list_empty(&folio_list))
|
| 3444 | prep_and_add_allocated_folios(h, &folio_list);
|
| 3445 |
|
| 3446 | if (i == h->max_huge_pages_node[nid])
|
| 3447 | return;
|
| 3448 |
|
| 3449 | string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
|
| 3450 | pr_warn("HugeTLB: allocating %u of page size %s failed node%d. Only allocated %lu hugepages.\n",
|
| 3451 | h->max_huge_pages_node[nid], buf, nid, i);
|
| 3452 | h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
|
| 3453 | h->max_huge_pages_node[nid] = i;
|
| 3454 | }
|
| 3455 |
|
| 3456 | static bool __init hugetlb_hstate_alloc_pages_specific_nodes(struct hstate *h)
|
| 3457 | {
|
| 3458 | int i;
|
| 3459 | bool node_specific_alloc = false;
|
| 3460 |
|
| 3461 | for_each_online_node(i) {
|
| 3462 | if (h->max_huge_pages_node[i] > 0) {
|
| 3463 | hugetlb_hstate_alloc_pages_onenode(h, i);
|
| 3464 | node_specific_alloc = true;
|
| 3465 | }
|
| 3466 | }
|
| 3467 |
|
| 3468 | return node_specific_alloc;
|
| 3469 | }
|
| 3470 |
|
| 3471 | static void __init hugetlb_hstate_alloc_pages_errcheck(unsigned long allocated, struct hstate *h)
|
| 3472 | {
|
| 3473 | if (allocated < h->max_huge_pages) {
|
| 3474 | char buf[32];
|
| 3475 |
|
| 3476 | string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
|
| 3477 | pr_warn("HugeTLB: allocating %lu of page size %s failed. Only allocated %lu hugepages.\n",
|
| 3478 | h->max_huge_pages, buf, allocated);
|
| 3479 | h->max_huge_pages = allocated;
|
| 3480 | }
|
| 3481 | }
|
| 3482 |
|
| 3483 | static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned long end, void *arg)
|
| 3484 | {
|
| 3485 | struct hstate *h = (struct hstate *)arg;
|
| 3486 | int i, num = end - start;
|
| 3487 | nodemask_t node_alloc_noretry;
|
| 3488 | LIST_HEAD(folio_list);
|
| 3489 | int next_node = first_online_node;
|
| 3490 |
|
| 3491 |
|
| 3492 | nodes_clear(node_alloc_noretry);
|
| 3493 |
|
| 3494 | for (i = 0; i < num; ++i) {
|
| 3495 | struct folio *folio;
|
| 3496 |
|
| 3497 | if (hugetlb_vmemmap_optimizable_size(h) &&
|
| 3498 | (si_mem_available() == 0) && !list_empty(&folio_list)) {
|
| 3499 | prep_and_add_allocated_folios(h, &folio_list);
|
| 3500 | INIT_LIST_HEAD(&folio_list);
|
| 3501 | }
|
| 3502 | folio = alloc_pool_huge_folio(h, &node_states[N_MEMORY],
|
| 3503 | &node_alloc_noretry, &next_node);
|
| 3504 | if (!folio)
|
| 3505 | break;
|
| 3506 |
|
| 3507 | list_move(&folio->lru, &folio_list);
|
| 3508 | cond_resched();
|
| 3509 | }
|
| 3510 |
|
| 3511 | prep_and_add_allocated_folios(h, &folio_list);
|
| 3512 | }
|
| 3513 |
|
| 3514 | static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
|
| 3515 | {
|
| 3516 | int nid;
|
| 3517 | unsigned long i;
|
| 3518 |
|
| 3519 | for (i = 0; i < h->max_huge_pages; ++i) {
|
| 3520 | if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
|
| 3521 | break;
|
| 3522 | cond_resched();
|
| 3523 | }
|
| 3524 |
|
| 3525 | for_each_node(nid)
|
| 3526 | i -= hugetlb_free_cross_zone_pages(h, nid);
|
| 3527 |
|
| 3528 | return i;
|
| 3529 | }
|
| 3530 |
|
| 3531 | static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
|
| 3532 | {
|
| 3533 | struct padata_mt_job job = {
|
| 3534 | .fn_arg = h,
|
| 3535 | .align = 1,
|
| 3536 | .numa_aware = true
|
| 3537 | };
|
| 3538 |
|
| 3539 | unsigned long jiffies_start;
|
| 3540 | unsigned long jiffies_end;
|
| 3541 | unsigned long remaining;
|
| 3542 |
|
| 3543 | job.thread_fn = hugetlb_pages_alloc_boot_node;
|
| 3544 |
|
| 3545 | |
| 3546 | * job.max_threads is 25% of the available cpu threads by default.
|
| 3547 | *
|
| 3548 | * On large servers with terabytes of memory, huge page allocation
|
| 3549 | * can consume a considerably amount of time.
|
| 3550 | *
|
| 3551 | * Tests below show how long it takes to allocate 1 TiB of memory with 2MiB huge pages.
|
| 3552 | * 2MiB huge pages. Using more threads can significantly improve allocation time.
|
| 3553 | *
|
| 3554 | * +-----------------------+-------+-------+-------+-------+-------+
|
| 3555 | * | threads | 8 | 16 | 32 | 64 | 128 |
|
| 3556 | * +-----------------------+-------+-------+-------+-------+-------+
|
| 3557 | * | skylake 144 cpus | 44s | 22s | 16s | 19s | 20s |
|
| 3558 | * | cascade lake 192 cpus | 39s | 20s | 11s | 10s | 9s |
|
| 3559 | * +-----------------------+-------+-------+-------+-------+-------+
|
| 3560 |
|
| 3561 | if (hugepage_allocation_threads == 0) {
|
| 3562 | hugepage_allocation_threads = num_online_cpus() / 4;
|
| 3563 | hugepage_allocation_threads = max(hugepage_allocation_threads, 1);
|
| 3564 | }
|
| 3565 |
|
| 3566 | job.max_threads = hugepage_allocation_threads;
|
| 3567 |
|
| 3568 | jiffies_start = jiffies;
|
| 3569 | do {
|
| 3570 | remaining = h->max_huge_pages - h->nr_huge_pages;
|
| 3571 |
|
| 3572 | job.start = h->nr_huge_pages;
|
| 3573 | job.size = remaining;
|
| 3574 | job.min_chunk = remaining / hugepage_allocation_threads;
|
| 3575 | padata_do_multithreaded(&job);
|
| 3576 |
|
| 3577 | if (h->nr_huge_pages == h->max_huge_pages)
|
| 3578 | break;
|
| 3579 |
|
| 3580 | |
| 3581 | * Retry only if the vmemmap optimization might have been able to free
|
| 3582 | * some memory back to the system.
|
| 3583 |
|
| 3584 | if (!hugetlb_vmemmap_optimizable(h))
|
| 3585 | break;
|
| 3586 |
|
| 3587 |
|
| 3588 | } while (remaining != (h->max_huge_pages - h->nr_huge_pages));
|
| 3589 |
|
| 3590 | jiffies_end = jiffies;
|
| 3591 |
|
| 3592 | pr_info("HugeTLB: allocation took %dms with hugepage_allocation_threads=%ld\n",
|
| 3593 | jiffies_to_msecs(jiffies_end - jiffies_start),
|
| 3594 | hugepage_allocation_threads);
|
| 3595 |
|
| 3596 | return h->nr_huge_pages;
|
| 3597 | }
|
| 3598 |
|
| 3599 | |
| 3600 | * NOTE: this routine is called in different contexts for gigantic and
|
| 3601 | * non-gigantic pages.
|
| 3602 | * - For gigantic pages, this is called early in the boot process and
|
| 3603 | * pages are allocated from memblock allocated or something similar.
|
| 3604 | * Gigantic pages are actually added to pools later with the routine
|
| 3605 | * hugetlb_bootmem_struct_page_init.
|
| 3606 | * - For non-gigantic pages, this is called later in the boot process after
|
| 3607 | * all of mm is up and functional. Pages are allocated from buddy and
|
| 3608 | * then added to hugetlb pools.
|
| 3609 |
|
| 3610 | static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
|
| 3611 | {
|
| 3612 | unsigned long allocated;
|
| 3613 |
|
| 3614 | |
| 3615 | * Skip gigantic hugepages allocation if early CMA
|
| 3616 | * reservations are not available.
|
| 3617 |
|
| 3618 | if (hstate_is_gigantic(h) && hugetlb_cma_total_size() &&
|
| 3619 | !hugetlb_early_cma(h)) {
|
| 3620 | pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
|
| 3621 | return;
|
| 3622 | }
|
| 3623 |
|
| 3624 | if (!h->max_huge_pages)
|
| 3625 | return;
|
| 3626 |
|
| 3627 |
|
| 3628 | if (hugetlb_hstate_alloc_pages_specific_nodes(h))
|
| 3629 | return;
|
| 3630 |
|
| 3631 |
|
| 3632 | if (hstate_is_gigantic(h))
|
| 3633 | allocated = hugetlb_gigantic_pages_alloc_boot(h);
|
| 3634 | else
|
| 3635 | allocated = hugetlb_pages_alloc_boot(h);
|
| 3636 |
|
| 3637 | hugetlb_hstate_alloc_pages_errcheck(allocated, h);
|
| 3638 | }
|
| 3639 |
|
| 3640 | static void __init hugetlb_init_hstates(void)
|
| 3641 | {
|
| 3642 | struct hstate *h, *h2;
|
| 3643 |
|
| 3644 | for_each_hstate(h) {
|
| 3645 | |
| 3646 | * Always reset to first_memory_node here, even if
|
| 3647 | * next_nid_to_alloc was set before - we can't
|
| 3648 | * reference hugetlb_bootmem_nodes after init, and
|
| 3649 | * first_memory_node is right for all further allocations.
|
| 3650 |
|
| 3651 | h->next_nid_to_alloc = first_memory_node;
|
| 3652 | h->next_nid_to_free = first_memory_node;
|
| 3653 |
|
| 3654 |
|
| 3655 | if (!hstate_is_gigantic(h))
|
| 3656 | hugetlb_hstate_alloc_pages(h);
|
| 3657 |
|
| 3658 | |
| 3659 | * Set demote order for each hstate. Note that
|
| 3660 | * h->demote_order is initially 0.
|
| 3661 | * - We can not demote gigantic pages if runtime freeing
|
| 3662 | * is not supported, so skip this.
|
| 3663 | * - If CMA allocation is possible, we can not demote
|
| 3664 | * HUGETLB_PAGE_ORDER or smaller size pages.
|
| 3665 |
|
| 3666 | if (hstate_is_gigantic_no_runtime(h))
|
| 3667 | continue;
|
| 3668 | if (hugetlb_cma_total_size() && h->order <= HUGETLB_PAGE_ORDER)
|
| 3669 | continue;
|
| 3670 | for_each_hstate(h2) {
|
| 3671 | if (h2 == h)
|
| 3672 | continue;
|
| 3673 | if (h2->order < h->order &&
|
| 3674 | h2->order > h->demote_order)
|
| 3675 | h->demote_order = h2->order;
|
| 3676 | }
|
| 3677 | }
|
| 3678 | }
|
| 3679 |
|
| 3680 | static void __init report_hugepages(void)
|
| 3681 | {
|
| 3682 | struct hstate *h;
|
| 3683 |
|
| 3684 | for_each_hstate(h) {
|
| 3685 | char buf[32];
|
| 3686 |
|
| 3687 | string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
|
| 3688 | pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
|
| 3689 | buf, h->nr_huge_pages);
|
| 3690 | pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
|
| 3691 | hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
|
| 3692 | }
|
| 3693 | }
|
| 3694 |
|
| 3695 | #ifdef CONFIG_HIGHMEM
|
| 3696 | static void try_to_free_low(struct hstate *h, unsigned long count,
|
| 3697 | nodemask_t *nodes_allowed)
|
| 3698 | {
|
| 3699 | int i;
|
| 3700 | LIST_HEAD(page_list);
|
| 3701 |
|
| 3702 | lockdep_assert_held(&hugetlb_lock);
|
| 3703 | if (hstate_is_gigantic(h))
|
| 3704 | return;
|
| 3705 |
|
| 3706 | |
| 3707 | * Collect pages to be freed on a list, and free after dropping lock
|
| 3708 |
|
| 3709 | for_each_node_mask(i, *nodes_allowed) {
|
| 3710 | struct folio *folio, *next;
|
| 3711 | struct list_head *freel = &h->hugepage_freelists[i];
|
| 3712 | list_for_each_entry_safe(folio, next, freel, lru) {
|
| 3713 | if (count >= h->nr_huge_pages)
|
| 3714 | goto out;
|
| 3715 | if (folio_test_highmem(folio))
|
| 3716 | continue;
|
| 3717 | remove_hugetlb_folio(h, folio, false);
|
| 3718 | list_add(&folio->lru, &page_list);
|
| 3719 | }
|
| 3720 | }
|
| 3721 |
|
| 3722 | out:
|
| 3723 | spin_unlock_irq(&hugetlb_lock);
|
| 3724 | update_and_free_pages_bulk(h, &page_list);
|
| 3725 | spin_lock_irq(&hugetlb_lock);
|
| 3726 | }
|
| 3727 | #else
|
| 3728 | static inline void try_to_free_low(struct hstate *h, unsigned long count,
|
| 3729 | nodemask_t *nodes_allowed)
|
| 3730 | {
|
| 3731 | }
|
| 3732 | #endif
|
| 3733 |
|
| 3734 | |
| 3735 | * Increment or decrement surplus_huge_pages. Keep node-specific counters
|
| 3736 | * balanced by operating on them in a round-robin fashion.
|
| 3737 | * Returns 1 if an adjustment was made.
|
| 3738 |
|
| 3739 | static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
|
| 3740 | int delta)
|
| 3741 | {
|
| 3742 | int nr_nodes, node;
|
| 3743 |
|
| 3744 | lockdep_assert_held(&hugetlb_lock);
|
| 3745 | VM_BUG_ON(delta != -1 && delta != 1);
|
| 3746 |
|
| 3747 | if (delta < 0) {
|
| 3748 | for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, nodes_allowed) {
|
| 3749 | if (h->surplus_huge_pages_node[node])
|
| 3750 | goto found;
|
| 3751 | }
|
| 3752 | } else {
|
| 3753 | for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
|
| 3754 | if (h->surplus_huge_pages_node[node] <
|
| 3755 | h->nr_huge_pages_node[node])
|
| 3756 | goto found;
|
| 3757 | }
|
| 3758 | }
|
| 3759 | return 0;
|
| 3760 |
|
| 3761 | found:
|
| 3762 | h->surplus_huge_pages += delta;
|
| 3763 | h->surplus_huge_pages_node[node] += delta;
|
| 3764 | return 1;
|
| 3765 | }
|
| 3766 |
|
| 3767 | #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
|
| 3768 | static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
|
| 3769 | nodemask_t *nodes_allowed)
|
| 3770 | {
|
| 3771 | unsigned long persistent_free_count;
|
| 3772 | unsigned long min_count;
|
| 3773 | unsigned long allocated;
|
| 3774 | struct folio *folio;
|
| 3775 | LIST_HEAD(page_list);
|
| 3776 | NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
|
| 3777 |
|
| 3778 | |
| 3779 | * Bit mask controlling how hard we retry per-node allocations.
|
| 3780 | * If we can not allocate the bit mask, do not attempt to allocate
|
| 3781 | * the requested huge pages.
|
| 3782 |
|
| 3783 | if (node_alloc_noretry)
|
| 3784 | nodes_clear(*node_alloc_noretry);
|
| 3785 | else
|
| 3786 | return -ENOMEM;
|
| 3787 |
|
| 3788 | |
| 3789 | * resize_lock mutex prevents concurrent adjustments to number of
|
| 3790 | * pages in hstate via the proc/sysfs interfaces.
|
| 3791 |
|
| 3792 | mutex_lock(&h->resize_lock);
|
| 3793 | flush_free_hpage_work(h);
|
| 3794 | spin_lock_irq(&hugetlb_lock);
|
| 3795 |
|
| 3796 | |
| 3797 | * Check for a node specific request.
|
| 3798 | * Changing node specific huge page count may require a corresponding
|
| 3799 | * change to the global count. In any case, the passed node mask
|
| 3800 | * (nodes_allowed) will restrict alloc/free to the specified node.
|
| 3801 |
|
| 3802 | if (nid != NUMA_NO_NODE) {
|
| 3803 | unsigned long old_count = count;
|
| 3804 |
|
| 3805 | count += persistent_huge_pages(h) -
|
| 3806 | (h->nr_huge_pages_node[nid] -
|
| 3807 | h->surplus_huge_pages_node[nid]);
|
| 3808 | |
| 3809 | * User may have specified a large count value which caused the
|
| 3810 | * above calculation to overflow. In this case, they wanted
|
| 3811 | * to allocate as many huge pages as possible. Set count to
|
| 3812 | * largest possible value to align with their intention.
|
| 3813 |
|
| 3814 | if (count < old_count)
|
| 3815 | count = ULONG_MAX;
|
| 3816 | }
|
| 3817 |
|
| 3818 | |
| 3819 | * Gigantic pages runtime allocation depend on the capability for large
|
| 3820 | * page range allocation.
|
| 3821 | * If the system does not provide this feature, return an error when
|
| 3822 | * the user tries to allocate gigantic pages but let the user free the
|
| 3823 | * boottime allocated gigantic pages.
|
| 3824 |
|
| 3825 | if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
|
| 3826 | if (count > persistent_huge_pages(h)) {
|
| 3827 | spin_unlock_irq(&hugetlb_lock);
|
| 3828 | mutex_unlock(&h->resize_lock);
|
| 3829 | NODEMASK_FREE(node_alloc_noretry);
|
| 3830 | return -EINVAL;
|
| 3831 | }
|
| 3832 |
|
| 3833 | }
|
| 3834 |
|
| 3835 | |
| 3836 | * Increase the pool size
|
| 3837 | * First take pages out of surplus state. Then make up the
|
| 3838 | * remaining difference by allocating fresh huge pages.
|
| 3839 | *
|
| 3840 | * We might race with alloc_surplus_hugetlb_folio() here and be unable
|
| 3841 | * to convert a surplus huge page to a normal huge page. That is
|
| 3842 | * not critical, though, it just means the overall size of the
|
| 3843 | * pool might be one hugepage larger than it needs to be, but
|
| 3844 | * within all the constraints specified by the sysctls.
|
| 3845 |
|
| 3846 | while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
|
| 3847 | if (!adjust_pool_surplus(h, nodes_allowed, -1))
|
| 3848 | break;
|
| 3849 | }
|
| 3850 |
|
| 3851 | allocated = 0;
|
| 3852 | while (count > (persistent_huge_pages(h) + allocated)) {
|
| 3853 | |
| 3854 | * If this allocation races such that we no longer need the
|
| 3855 | * page, free_huge_folio will handle it by freeing the page
|
| 3856 | * and reducing the surplus.
|
| 3857 |
|
| 3858 | spin_unlock_irq(&hugetlb_lock);
|
| 3859 |
|
| 3860 |
|
| 3861 | cond_resched();
|
| 3862 |
|
| 3863 | folio = alloc_pool_huge_folio(h, nodes_allowed,
|
| 3864 | node_alloc_noretry,
|
| 3865 | &h->next_nid_to_alloc);
|
| 3866 | if (!folio) {
|
| 3867 | prep_and_add_allocated_folios(h, &page_list);
|
| 3868 | spin_lock_irq(&hugetlb_lock);
|
| 3869 | goto out;
|
| 3870 | }
|
| 3871 |
|
| 3872 | list_add(&folio->lru, &page_list);
|
| 3873 | allocated++;
|
| 3874 |
|
| 3875 |
|
| 3876 | if (signal_pending(current)) {
|
| 3877 | prep_and_add_allocated_folios(h, &page_list);
|
| 3878 | spin_lock_irq(&hugetlb_lock);
|
| 3879 | goto out;
|
| 3880 | }
|
| 3881 |
|
| 3882 | spin_lock_irq(&hugetlb_lock);
|
| 3883 | }
|
| 3884 |
|
| 3885 |
|
| 3886 | if (!list_empty(&page_list)) {
|
| 3887 | spin_unlock_irq(&hugetlb_lock);
|
| 3888 | prep_and_add_allocated_folios(h, &page_list);
|
| 3889 | spin_lock_irq(&hugetlb_lock);
|
| 3890 | }
|
| 3891 |
|
| 3892 | |
| 3893 | * Decrease the pool size
|
| 3894 | * First return free pages to the buddy allocator (being careful
|
| 3895 | * to keep enough around to satisfy reservations). Then place
|
| 3896 | * pages into surplus state as needed so the pool will shrink
|
| 3897 | * to the desired size as pages become free.
|
| 3898 | *
|
| 3899 | * By placing pages into the surplus state independent of the
|
| 3900 | * overcommit value, we are allowing the surplus pool size to
|
| 3901 | * exceed overcommit. There are few sane options here. Since
|
| 3902 | * alloc_surplus_hugetlb_folio() is checking the global counter,
|
| 3903 | * though, we'll note that we're not allowed to exceed surplus
|
| 3904 | * and won't grow the pool anywhere else. Not until one of the
|
| 3905 | * sysctls are changed, or the surplus pages go out of use.
|
| 3906 | *
|
| 3907 | * min_count is the expected number of persistent pages, we
|
| 3908 | * shouldn't calculate min_count by using
|
| 3909 | * resv_huge_pages + persistent_huge_pages() - free_huge_pages,
|
| 3910 | * because there may exist free surplus huge pages, and this will
|
| 3911 | * lead to subtracting twice. Free surplus huge pages come from HVO
|
| 3912 | * failing to restore vmemmap, see comments in the callers of
|
| 3913 | * hugetlb_vmemmap_restore_folio(). Thus, we should calculate
|
| 3914 | * persistent free count first.
|
| 3915 |
|
| 3916 | persistent_free_count = h->free_huge_pages;
|
| 3917 | if (h->free_huge_pages > persistent_huge_pages(h)) {
|
| 3918 | if (h->free_huge_pages > h->surplus_huge_pages)
|
| 3919 | persistent_free_count -= h->surplus_huge_pages;
|
| 3920 | else
|
| 3921 | persistent_free_count = 0;
|
| 3922 | }
|
| 3923 | min_count = h->resv_huge_pages + persistent_huge_pages(h) - persistent_free_count;
|
| 3924 | min_count = max(count, min_count);
|
| 3925 | try_to_free_low(h, min_count, nodes_allowed);
|
| 3926 |
|
| 3927 | |
| 3928 | * Collect pages to be removed on list without dropping lock
|
| 3929 |
|
| 3930 | while (min_count < persistent_huge_pages(h)) {
|
| 3931 | folio = remove_pool_hugetlb_folio(h, nodes_allowed, 0);
|
| 3932 | if (!folio)
|
| 3933 | break;
|
| 3934 |
|
| 3935 | list_add(&folio->lru, &page_list);
|
| 3936 | }
|
| 3937 |
|
| 3938 | spin_unlock_irq(&hugetlb_lock);
|
| 3939 | update_and_free_pages_bulk(h, &page_list);
|
| 3940 | flush_free_hpage_work(h);
|
| 3941 | spin_lock_irq(&hugetlb_lock);
|
| 3942 |
|
| 3943 | while (count < persistent_huge_pages(h)) {
|
| 3944 | if (!adjust_pool_surplus(h, nodes_allowed, 1))
|
| 3945 | break;
|
| 3946 | }
|
| 3947 | out:
|
| 3948 | h->max_huge_pages = persistent_huge_pages(h);
|
| 3949 | spin_unlock_irq(&hugetlb_lock);
|
| 3950 | mutex_unlock(&h->resize_lock);
|
| 3951 |
|
| 3952 | NODEMASK_FREE(node_alloc_noretry);
|
| 3953 |
|
| 3954 | return 0;
|
| 3955 | }
|
| 3956 |
|
| 3957 | static long demote_free_hugetlb_folios(struct hstate *src, struct hstate *dst,
|
| 3958 | struct list_head *src_list)
|
| 3959 | {
|
| 3960 | long rc;
|
| 3961 | struct folio *folio, *next;
|
| 3962 | LIST_HEAD(dst_list);
|
| 3963 | LIST_HEAD(ret_list);
|
| 3964 |
|
| 3965 | rc = hugetlb_vmemmap_restore_folios(src, src_list, &ret_list);
|
| 3966 | list_splice_init(&ret_list, src_list);
|
| 3967 |
|
| 3968 | |
| 3969 | * Taking target hstate mutex synchronizes with set_max_huge_pages.
|
| 3970 | * Without the mutex, pages added to target hstate could be marked
|
| 3971 | * as surplus.
|
| 3972 | *
|
| 3973 | * Note that we already hold src->resize_lock. To prevent deadlock,
|
| 3974 | * use the convention of always taking larger size hstate mutex first.
|
| 3975 |
|
| 3976 | mutex_lock(&dst->resize_lock);
|
| 3977 |
|
| 3978 | list_for_each_entry_safe(folio, next, src_list, lru) {
|
| 3979 | int i;
|
| 3980 | bool cma;
|
| 3981 |
|
| 3982 | if (folio_test_hugetlb_vmemmap_optimized(folio))
|
| 3983 | continue;
|
| 3984 |
|
| 3985 | cma = folio_test_hugetlb_cma(folio);
|
| 3986 |
|
| 3987 | list_del(&folio->lru);
|
| 3988 |
|
| 3989 | split_page_owner(&folio->page, huge_page_order(src), huge_page_order(dst));
|
| 3990 | pgalloc_tag_split(folio, huge_page_order(src), huge_page_order(dst));
|
| 3991 |
|
| 3992 | for (i = 0; i < pages_per_huge_page(src); i += pages_per_huge_page(dst)) {
|
| 3993 | struct page *page = folio_page(folio, i);
|
| 3994 |
|
| 3995 | struct folio *new_folio = (struct folio *)page;
|
| 3996 |
|
| 3997 | clear_compound_head(page);
|
| 3998 | prep_compound_page(page, dst->order);
|
| 3999 |
|
| 4000 | new_folio->mapping = NULL;
|
| 4001 | init_new_hugetlb_folio(new_folio);
|
| 4002 |
|
| 4003 | if (cma)
|
| 4004 | folio_set_hugetlb_cma(new_folio);
|
| 4005 | list_add(&new_folio->lru, &dst_list);
|
| 4006 | }
|
| 4007 | }
|
| 4008 |
|
| 4009 | prep_and_add_allocated_folios(dst, &dst_list);
|
| 4010 |
|
| 4011 | mutex_unlock(&dst->resize_lock);
|
| 4012 |
|
| 4013 | return rc;
|
| 4014 | }
|
| 4015 |
|
| 4016 | long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
|
| 4017 | unsigned long nr_to_demote)
|
| 4018 | __must_hold(&hugetlb_lock)
|
| 4019 | {
|
| 4020 | int nr_nodes, node;
|
| 4021 | struct hstate *dst;
|
| 4022 | long rc = 0;
|
| 4023 | long nr_demoted = 0;
|
| 4024 |
|
| 4025 | lockdep_assert_held(&hugetlb_lock);
|
| 4026 |
|
| 4027 |
|
| 4028 | if (!src->demote_order) {
|
| 4029 | pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
|
| 4030 | return -EINVAL;
|
| 4031 | }
|
| 4032 | dst = size_to_hstate(PAGE_SIZE << src->demote_order);
|
| 4033 |
|
| 4034 | for_each_node_mask_to_free(src, nr_nodes, node, nodes_allowed) {
|
| 4035 | LIST_HEAD(list);
|
| 4036 | struct folio *folio, *next;
|
| 4037 |
|
| 4038 | list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) {
|
| 4039 | if (folio_test_hwpoison(folio))
|
| 4040 | continue;
|
| 4041 |
|
| 4042 | remove_hugetlb_folio(src, folio, false);
|
| 4043 | list_add(&folio->lru, &list);
|
| 4044 |
|
| 4045 | if (++nr_demoted == nr_to_demote)
|
| 4046 | break;
|
| 4047 | }
|
| 4048 |
|
| 4049 | spin_unlock_irq(&hugetlb_lock);
|
| 4050 |
|
| 4051 | rc = demote_free_hugetlb_folios(src, dst, &list);
|
| 4052 |
|
| 4053 | spin_lock_irq(&hugetlb_lock);
|
| 4054 |
|
| 4055 | list_for_each_entry_safe(folio, next, &list, lru) {
|
| 4056 | list_del(&folio->lru);
|
| 4057 | add_hugetlb_folio(src, folio, false);
|
| 4058 |
|
| 4059 | nr_demoted--;
|
| 4060 | }
|
| 4061 |
|
| 4062 | if (rc < 0 || nr_demoted == nr_to_demote)
|
| 4063 | break;
|
| 4064 | }
|
| 4065 |
|
| 4066 | |
| 4067 | * Not absolutely necessary, but for consistency update max_huge_pages
|
| 4068 | * based on pool changes for the demoted page.
|
| 4069 |
|
| 4070 | src->max_huge_pages -= nr_demoted;
|
| 4071 | dst->max_huge_pages += nr_demoted << (huge_page_order(src) - huge_page_order(dst));
|
| 4072 |
|
| 4073 | if (rc < 0)
|
| 4074 | return rc;
|
| 4075 |
|
| 4076 | if (nr_demoted)
|
| 4077 | return nr_demoted;
|
| 4078 | |
| 4079 | * Only way to get here is if all pages on free lists are poisoned.
|
| 4080 | * Return -EBUSY so that caller will not retry.
|
| 4081 |
|
| 4082 | return -EBUSY;
|
| 4083 | }
|
| 4084 |
|
| 4085 | ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
|
| 4086 | struct hstate *h, int nid,
|
| 4087 | unsigned long count, size_t len)
|
| 4088 | {
|
| 4089 | int err;
|
| 4090 | nodemask_t nodes_allowed, *n_mask;
|
| 4091 |
|
| 4092 | if (hstate_is_gigantic_no_runtime(h))
|
| 4093 | return -EINVAL;
|
| 4094 |
|
| 4095 | if (nid == NUMA_NO_NODE) {
|
| 4096 | |
| 4097 | * global hstate attribute
|
| 4098 |
|
| 4099 | if (!(obey_mempolicy &&
|
| 4100 | init_nodemask_of_mempolicy(&nodes_allowed)))
|
| 4101 | n_mask = &node_states[N_MEMORY];
|
| 4102 | else
|
| 4103 | n_mask = &nodes_allowed;
|
| 4104 | } else {
|
| 4105 | |
| 4106 | * Node specific request. count adjustment happens in
|
| 4107 | * set_max_huge_pages() after acquiring hugetlb_lock.
|
| 4108 |
|
| 4109 | init_nodemask_of_node(&nodes_allowed, nid);
|
| 4110 | n_mask = &nodes_allowed;
|
| 4111 | }
|
| 4112 |
|
| 4113 | err = set_max_huge_pages(h, count, nid, n_mask);
|
| 4114 |
|
| 4115 | return err ? err : len;
|
| 4116 | }
|
| 4117 |
|
| 4118 | static int __init hugetlb_init(void)
|
| 4119 | {
|
| 4120 | int i;
|
| 4121 |
|
| 4122 | BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
|
| 4123 | __NR_HPAGEFLAGS);
|
| 4124 | BUILD_BUG_ON_INVALID(HUGETLB_PAGE_ORDER > MAX_FOLIO_ORDER);
|
| 4125 |
|
| 4126 | if (!hugepages_supported()) {
|
| 4127 | if (hugetlb_max_hstate || default_hstate_max_huge_pages)
|
| 4128 | pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
|
| 4129 | return 0;
|
| 4130 | }
|
| 4131 |
|
| 4132 | |
| 4133 | * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists. Some
|
| 4134 | * architectures depend on setup being done here.
|
| 4135 |
|
| 4136 | hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
|
| 4137 | if (!parsed_default_hugepagesz) {
|
| 4138 | |
| 4139 | * If we did not parse a default huge page size, set
|
| 4140 | * default_hstate_idx to HPAGE_SIZE hstate. And, if the
|
| 4141 | * number of huge pages for this default size was implicitly
|
| 4142 | * specified, set that here as well.
|
| 4143 | * Note that the implicit setting will overwrite an explicit
|
| 4144 | * setting. A warning will be printed in this case.
|
| 4145 |
|
| 4146 | default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
|
| 4147 | if (default_hstate_max_huge_pages) {
|
| 4148 | if (default_hstate.max_huge_pages) {
|
| 4149 | char buf[32];
|
| 4150 |
|
| 4151 | string_get_size(huge_page_size(&default_hstate),
|
| 4152 | 1, STRING_UNITS_2, buf, 32);
|
| 4153 | pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
|
| 4154 | default_hstate.max_huge_pages, buf);
|
| 4155 | pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
|
| 4156 | default_hstate_max_huge_pages);
|
| 4157 | }
|
| 4158 | default_hstate.max_huge_pages =
|
| 4159 | default_hstate_max_huge_pages;
|
| 4160 |
|
| 4161 | for_each_online_node(i)
|
| 4162 | default_hstate.max_huge_pages_node[i] =
|
| 4163 | default_hugepages_in_node[i];
|
| 4164 | }
|
| 4165 | }
|
| 4166 |
|
| 4167 | hugetlb_init_hstates();
|
| 4168 | report_hugepages();
|
| 4169 |
|
| 4170 | hugetlb_sysfs_init();
|
| 4171 | hugetlb_cgroup_file_init();
|
| 4172 | hugetlb_sysctl_init();
|
| 4173 |
|
| 4174 | #ifdef CONFIG_SMP
|
| 4175 | num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
|
| 4176 | #else
|
| 4177 | num_fault_mutexes = 1;
|
| 4178 | #endif
|
| 4179 | hugetlb_fault_mutex_table =
|
| 4180 | kmalloc_objs(struct mutex, num_fault_mutexes);
|
| 4181 | BUG_ON(!hugetlb_fault_mutex_table);
|
| 4182 |
|
| 4183 | for (i = 0; i < num_fault_mutexes; i++)
|
| 4184 | mutex_init(&hugetlb_fault_mutex_table[i]);
|
| 4185 | return 0;
|
| 4186 | }
|
| 4187 | subsys_initcall(hugetlb_init);
|
| 4188 |
|
| 4189 |
|
| 4190 | bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
|
| 4191 | {
|
| 4192 | return size == HPAGE_SIZE;
|
| 4193 | }
|
| 4194 |
|
| 4195 | void __init hugetlb_add_hstate(unsigned int order)
|
| 4196 | {
|
| 4197 | struct hstate *h;
|
| 4198 | unsigned long i;
|
| 4199 |
|
| 4200 | if (size_to_hstate(PAGE_SIZE << order)) {
|
| 4201 | return;
|
| 4202 | }
|
| 4203 | BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
|
| 4204 | BUG_ON(order < order_base_2(__NR_USED_SUBPAGE));
|
| 4205 | WARN_ON(order > MAX_FOLIO_ORDER);
|
| 4206 | h = &hstates[hugetlb_max_hstate++];
|
| 4207 | __mutex_init(&h->resize_lock, "resize mutex", &h->resize_key);
|
| 4208 | h->order = order;
|
| 4209 | h->mask = ~(huge_page_size(h) - 1);
|
| 4210 | for (i = 0; i < MAX_NUMNODES; ++i)
|
| 4211 | INIT_LIST_HEAD(&h->hugepage_freelists[i]);
|
| 4212 | INIT_LIST_HEAD(&h->hugepage_activelist);
|
| 4213 | snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
|
| 4214 | huge_page_size(h)/SZ_1K);
|
| 4215 |
|
| 4216 | parsed_hstate = h;
|
| 4217 | }
|
| 4218 |
|
| 4219 | bool __init __weak hugetlb_node_alloc_supported(void)
|
| 4220 | {
|
| 4221 | return true;
|
| 4222 | }
|
| 4223 |
|
| 4224 | static void __init hugepages_clear_pages_in_node(void)
|
| 4225 | {
|
| 4226 | if (!hugetlb_max_hstate) {
|
| 4227 | default_hstate_max_huge_pages = 0;
|
| 4228 | memset(default_hugepages_in_node, 0,
|
| 4229 | sizeof(default_hugepages_in_node));
|
| 4230 | } else {
|
| 4231 | parsed_hstate->max_huge_pages = 0;
|
| 4232 | memset(parsed_hstate->max_huge_pages_node, 0,
|
| 4233 | sizeof(parsed_hstate->max_huge_pages_node));
|
| 4234 | }
|
| 4235 | }
|
| 4236 |
|
| 4237 | static __init int hugetlb_add_param(char *s, int (*setup)(char *))
|
| 4238 | {
|
| 4239 | size_t len;
|
| 4240 | char *p;
|
| 4241 |
|
| 4242 | if (!s)
|
| 4243 | return -EINVAL;
|
| 4244 |
|
| 4245 | if (hugetlb_param_index >= HUGE_MAX_CMDLINE_ARGS)
|
| 4246 | return -EINVAL;
|
| 4247 |
|
| 4248 | len = strlen(s) + 1;
|
| 4249 | if (len + hstate_cmdline_index > sizeof(hstate_cmdline_buf))
|
| 4250 | return -EINVAL;
|
| 4251 |
|
| 4252 | p = &hstate_cmdline_buf[hstate_cmdline_index];
|
| 4253 | memcpy(p, s, len);
|
| 4254 | hstate_cmdline_index += len;
|
| 4255 |
|
| 4256 | hugetlb_params[hugetlb_param_index].val = p;
|
| 4257 | hugetlb_params[hugetlb_param_index].setup = setup;
|
| 4258 |
|
| 4259 | hugetlb_param_index++;
|
| 4260 |
|
| 4261 | return 0;
|
| 4262 | }
|
| 4263 |
|
| 4264 | static __init void hugetlb_parse_params(void)
|
| 4265 | {
|
| 4266 | int i;
|
| 4267 | struct hugetlb_cmdline *hcp;
|
| 4268 |
|
| 4269 | for (i = 0; i < hugetlb_param_index; i++) {
|
| 4270 | hcp = &hugetlb_params[i];
|
| 4271 |
|
| 4272 | hcp->setup(hcp->val);
|
| 4273 | }
|
| 4274 |
|
| 4275 | hugetlb_cma_validate_params();
|
| 4276 | }
|
| 4277 |
|
| 4278 | |
| 4279 | * hugepages command line processing
|
| 4280 | * hugepages normally follows a valid hugepagsz or default_hugepagsz
|
| 4281 | * specification. If not, ignore the hugepages value. hugepages can also
|
| 4282 | * be the first huge page command line option in which case it implicitly
|
| 4283 | * specifies the number of huge pages for the default size.
|
| 4284 |
|
| 4285 | static int __init hugepages_setup(char *s)
|
| 4286 | {
|
| 4287 | unsigned long *mhp;
|
| 4288 | static unsigned long *last_mhp;
|
| 4289 | int node = NUMA_NO_NODE;
|
| 4290 | int count;
|
| 4291 | unsigned long tmp;
|
| 4292 | char *p = s;
|
| 4293 |
|
| 4294 | if (!hugepages_supported()) {
|
| 4295 | pr_warn("HugeTLB: hugepages unsupported, ignoring hugepages=%s cmdline\n", s);
|
| 4296 | return 0;
|
| 4297 | }
|
| 4298 |
|
| 4299 | if (!parsed_valid_hugepagesz) {
|
| 4300 | pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
|
| 4301 | parsed_valid_hugepagesz = true;
|
| 4302 | return -EINVAL;
|
| 4303 | }
|
| 4304 |
|
| 4305 | |
| 4306 | * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
|
| 4307 | * yet, so this hugepages= parameter goes to the "default hstate".
|
| 4308 | * Otherwise, it goes with the previously parsed hugepagesz or
|
| 4309 | * default_hugepagesz.
|
| 4310 |
|
| 4311 | else if (!hugetlb_max_hstate)
|
| 4312 | mhp = &default_hstate_max_huge_pages;
|
| 4313 | else
|
| 4314 | mhp = &parsed_hstate->max_huge_pages;
|
| 4315 |
|
| 4316 | if (mhp == last_mhp) {
|
| 4317 | pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
|
| 4318 | return 1;
|
| 4319 | }
|
| 4320 |
|
| 4321 | while (*p) {
|
| 4322 | count = 0;
|
| 4323 | if (sscanf(p, "%lu%n", &tmp, &count) != 1)
|
| 4324 | goto invalid;
|
| 4325 |
|
| 4326 | if (p[count] == ':') {
|
| 4327 | if (!hugetlb_node_alloc_supported()) {
|
| 4328 | pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
|
| 4329 | return 1;
|
| 4330 | }
|
| 4331 | if (tmp >= MAX_NUMNODES || !node_online(tmp))
|
| 4332 | goto invalid;
|
| 4333 | node = array_index_nospec(tmp, MAX_NUMNODES);
|
| 4334 | p += count + 1;
|
| 4335 |
|
| 4336 | if (sscanf(p, "%lu%n", &tmp, &count) != 1)
|
| 4337 | goto invalid;
|
| 4338 | if (!hugetlb_max_hstate)
|
| 4339 | default_hugepages_in_node[node] = tmp;
|
| 4340 | else
|
| 4341 | parsed_hstate->max_huge_pages_node[node] = tmp;
|
| 4342 | *mhp += tmp;
|
| 4343 |
|
| 4344 | if (p[count] == ',')
|
| 4345 | p += count + 1;
|
| 4346 | else
|
| 4347 | break;
|
| 4348 | } else {
|
| 4349 | if (p != s)
|
| 4350 | goto invalid;
|
| 4351 | *mhp = tmp;
|
| 4352 | break;
|
| 4353 | }
|
| 4354 | }
|
| 4355 |
|
| 4356 | last_mhp = mhp;
|
| 4357 |
|
| 4358 | return 0;
|
| 4359 |
|
| 4360 | invalid:
|
| 4361 | pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
|
| 4362 | hugepages_clear_pages_in_node();
|
| 4363 | return -EINVAL;
|
| 4364 | }
|
| 4365 | hugetlb_early_param("hugepages", hugepages_setup);
|
| 4366 |
|
| 4367 | |
| 4368 | * hugepagesz command line processing
|
| 4369 | * A specific huge page size can only be specified once with hugepagesz.
|
| 4370 | * hugepagesz is followed by hugepages on the command line. The global
|
| 4371 | * variable 'parsed_valid_hugepagesz' is used to determine if prior
|
| 4372 | * hugepagesz argument was valid.
|
| 4373 |
|
| 4374 | static int __init hugepagesz_setup(char *s)
|
| 4375 | {
|
| 4376 | unsigned long size;
|
| 4377 | struct hstate *h;
|
| 4378 |
|
| 4379 | if (!hugepages_supported()) {
|
| 4380 | pr_warn("HugeTLB: hugepages unsupported, ignoring hugepagesz=%s cmdline\n", s);
|
| 4381 | return 0;
|
| 4382 | }
|
| 4383 |
|
| 4384 | parsed_valid_hugepagesz = false;
|
| 4385 | size = (unsigned long)memparse(s, NULL);
|
| 4386 |
|
| 4387 | if (!arch_hugetlb_valid_size(size)) {
|
| 4388 | pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
|
| 4389 | return -EINVAL;
|
| 4390 | }
|
| 4391 |
|
| 4392 | h = size_to_hstate(size);
|
| 4393 | if (h) {
|
| 4394 | |
| 4395 | * hstate for this size already exists. This is normally
|
| 4396 | * an error, but is allowed if the existing hstate is the
|
| 4397 | * default hstate. More specifically, it is only allowed if
|
| 4398 | * the number of huge pages for the default hstate was not
|
| 4399 | * previously specified.
|
| 4400 |
|
| 4401 | if (!parsed_default_hugepagesz || h != &default_hstate ||
|
| 4402 | default_hstate.max_huge_pages) {
|
| 4403 | pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
|
| 4404 | return -EINVAL;
|
| 4405 | }
|
| 4406 |
|
| 4407 | |
| 4408 | * No need to call hugetlb_add_hstate() as hstate already
|
| 4409 | * exists. But, do set parsed_hstate so that a following
|
| 4410 | * hugepages= parameter will be applied to this hstate.
|
| 4411 |
|
| 4412 | parsed_hstate = h;
|
| 4413 | parsed_valid_hugepagesz = true;
|
| 4414 | return 0;
|
| 4415 | }
|
| 4416 |
|
| 4417 | hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
|
| 4418 | parsed_valid_hugepagesz = true;
|
| 4419 | return 0;
|
| 4420 | }
|
| 4421 | hugetlb_early_param("hugepagesz", hugepagesz_setup);
|
| 4422 |
|
| 4423 | |
| 4424 | * default_hugepagesz command line input
|
| 4425 | * Only one instance of default_hugepagesz allowed on command line.
|
| 4426 |
|
| 4427 | static int __init default_hugepagesz_setup(char *s)
|
| 4428 | {
|
| 4429 | unsigned long size;
|
| 4430 | int i;
|
| 4431 |
|
| 4432 | if (!hugepages_supported()) {
|
| 4433 | pr_warn("HugeTLB: hugepages unsupported, ignoring default_hugepagesz=%s cmdline\n",
|
| 4434 | s);
|
| 4435 | return 0;
|
| 4436 | }
|
| 4437 |
|
| 4438 | parsed_valid_hugepagesz = false;
|
| 4439 | if (parsed_default_hugepagesz) {
|
| 4440 | pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
|
| 4441 | return -EINVAL;
|
| 4442 | }
|
| 4443 |
|
| 4444 | size = (unsigned long)memparse(s, NULL);
|
| 4445 |
|
| 4446 | if (!arch_hugetlb_valid_size(size)) {
|
| 4447 | pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
|
| 4448 | return -EINVAL;
|
| 4449 | }
|
| 4450 |
|
| 4451 | hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
|
| 4452 | parsed_valid_hugepagesz = true;
|
| 4453 | parsed_default_hugepagesz = true;
|
| 4454 | default_hstate_idx = hstate_index(size_to_hstate(size));
|
| 4455 |
|
| 4456 | |
| 4457 | * The number of default huge pages (for this size) could have been
|
| 4458 | * specified as the first hugetlb parameter: hugepages=X. If so,
|
| 4459 | * then default_hstate_max_huge_pages is set. If the default huge
|
| 4460 | * page size is gigantic (> MAX_PAGE_ORDER), then the pages must be
|
| 4461 | * allocated here from bootmem allocator.
|
| 4462 |
|
| 4463 | if (default_hstate_max_huge_pages) {
|
| 4464 | default_hstate.max_huge_pages = default_hstate_max_huge_pages;
|
| 4465 | |
| 4466 | * Since this is an early parameter, we can't check
|
| 4467 | * NUMA node state yet, so loop through MAX_NUMNODES.
|
| 4468 |
|
| 4469 | for (i = 0; i < MAX_NUMNODES; i++) {
|
| 4470 | if (default_hugepages_in_node[i] != 0)
|
| 4471 | default_hstate.max_huge_pages_node[i] =
|
| 4472 | default_hugepages_in_node[i];
|
| 4473 | }
|
| 4474 | default_hstate_max_huge_pages = 0;
|
| 4475 | }
|
| 4476 |
|
| 4477 | return 0;
|
| 4478 | }
|
| 4479 | hugetlb_early_param("default_hugepagesz", default_hugepagesz_setup);
|
| 4480 |
|
| 4481 | void __init hugetlb_bootmem_set_nodes(void)
|
| 4482 | {
|
| 4483 | int i, nid;
|
| 4484 |
|
| 4485 | if (!nodes_empty(hugetlb_bootmem_nodes))
|
| 4486 | return;
|
| 4487 |
|
| 4488 | for_each_mem_pfn_range(i, MAX_NUMNODES, NULL, NULL, &nid)
|
| 4489 | node_set(nid, hugetlb_bootmem_nodes);
|
| 4490 | }
|
| 4491 |
|
| 4492 | void __init hugetlb_bootmem_alloc(void)
|
| 4493 | {
|
| 4494 | struct hstate *h;
|
| 4495 | int i;
|
| 4496 |
|
| 4497 | hugetlb_bootmem_set_nodes();
|
| 4498 |
|
| 4499 | for (i = 0; i < MAX_NUMNODES; i++)
|
| 4500 | INIT_LIST_HEAD(&huge_boot_pages[i]);
|
| 4501 |
|
| 4502 | hugetlb_parse_params();
|
| 4503 |
|
| 4504 | for_each_hstate(h) {
|
| 4505 | h->next_nid_to_alloc = first_online_node;
|
| 4506 |
|
| 4507 | if (hstate_is_gigantic(h))
|
| 4508 | hugetlb_hstate_alloc_pages(h);
|
| 4509 | }
|
| 4510 | }
|
| 4511 |
|
| 4512 | |
| 4513 | * hugepage_alloc_threads command line parsing.
|
| 4514 | *
|
| 4515 | * When set, use this specific number of threads for the boot
|
| 4516 | * allocation of hugepages.
|
| 4517 |
|
| 4518 | static int __init hugepage_alloc_threads_setup(char *s)
|
| 4519 | {
|
| 4520 | unsigned long allocation_threads;
|
| 4521 |
|
| 4522 | if (kstrtoul(s, 0, &allocation_threads) != 0)
|
| 4523 | return 1;
|
| 4524 |
|
| 4525 | if (allocation_threads == 0)
|
| 4526 | return 1;
|
| 4527 |
|
| 4528 | hugepage_allocation_threads = allocation_threads;
|
| 4529 |
|
| 4530 | return 1;
|
| 4531 | }
|
| 4532 | __setup("hugepage_alloc_threads=", hugepage_alloc_threads_setup);
|
| 4533 |
|
| 4534 | static unsigned int allowed_mems_nr(struct hstate *h)
|
| 4535 | {
|
| 4536 | int node;
|
| 4537 | unsigned int nr = 0;
|
| 4538 | nodemask_t *mbind_nodemask;
|
| 4539 | unsigned int *array = h->free_huge_pages_node;
|
| 4540 | gfp_t gfp_mask = htlb_alloc_mask(h);
|
| 4541 |
|
| 4542 | mbind_nodemask = policy_mbind_nodemask(gfp_mask);
|
| 4543 | for_each_node_mask(node, cpuset_current_mems_allowed) {
|
| 4544 | if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
|
| 4545 | nr += array[node];
|
| 4546 | }
|
| 4547 |
|
| 4548 | return nr;
|
| 4549 | }
|
| 4550 |
|
| 4551 | void hugetlb_report_meminfo(struct seq_file *m)
|
| 4552 | {
|
| 4553 | struct hstate *h;
|
| 4554 | unsigned long total = 0;
|
| 4555 |
|
| 4556 | if (!hugepages_supported())
|
| 4557 | return;
|
| 4558 |
|
| 4559 | for_each_hstate(h) {
|
| 4560 | unsigned long count = h->nr_huge_pages;
|
| 4561 |
|
| 4562 | total += huge_page_size(h) * count;
|
| 4563 |
|
| 4564 | if (h == &default_hstate)
|
| 4565 | seq_printf(m,
|
| 4566 | "HugePages_Total: %5lu\n"
|
| 4567 | "HugePages_Free: %5lu\n"
|
| 4568 | "HugePages_Rsvd: %5lu\n"
|
| 4569 | "HugePages_Surp: %5lu\n"
|
| 4570 | "Hugepagesize: %8lu kB\n",
|
| 4571 | count,
|
| 4572 | h->free_huge_pages,
|
| 4573 | h->resv_huge_pages,
|
| 4574 | h->surplus_huge_pages,
|
| 4575 | huge_page_size(h) / SZ_1K);
|
| 4576 | }
|
| 4577 |
|
| 4578 | seq_printf(m, "Hugetlb: %8lu kB\n", total / SZ_1K);
|
| 4579 | }
|
| 4580 |
|
| 4581 | int hugetlb_report_node_meminfo(char *buf, int len, int nid)
|
| 4582 | {
|
| 4583 | struct hstate *h = &default_hstate;
|
| 4584 |
|
| 4585 | if (!hugepages_supported())
|
| 4586 | return 0;
|
| 4587 |
|
| 4588 | return sysfs_emit_at(buf, len,
|
| 4589 | "Node %d HugePages_Total: %5u\n"
|
| 4590 | "Node %d HugePages_Free: %5u\n"
|
| 4591 | "Node %d HugePages_Surp: %5u\n",
|
| 4592 | nid, h->nr_huge_pages_node[nid],
|
| 4593 | nid, h->free_huge_pages_node[nid],
|
| 4594 | nid, h->surplus_huge_pages_node[nid]);
|
| 4595 | }
|
| 4596 |
|
| 4597 | void hugetlb_show_meminfo_node(int nid)
|
| 4598 | {
|
| 4599 | struct hstate *h;
|
| 4600 |
|
| 4601 | if (!hugepages_supported())
|
| 4602 | return;
|
| 4603 |
|
| 4604 | for_each_hstate(h)
|
| 4605 | printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
|
| 4606 | nid,
|
| 4607 | h->nr_huge_pages_node[nid],
|
| 4608 | h->free_huge_pages_node[nid],
|
| 4609 | h->surplus_huge_pages_node[nid],
|
| 4610 | huge_page_size(h) / SZ_1K);
|
| 4611 | }
|
| 4612 |
|
| 4613 | void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
|
| 4614 | {
|
| 4615 | seq_printf(m, "HugetlbPages:\t%8lu kB\n",
|
| 4616 | K(atomic_long_read(&mm->hugetlb_usage)));
|
| 4617 | }
|
| 4618 |
|
| 4619 |
|
| 4620 | unsigned long hugetlb_total_pages(void)
|
| 4621 | {
|
| 4622 | struct hstate *h;
|
| 4623 | unsigned long nr_total_pages = 0;
|
| 4624 |
|
| 4625 | for_each_hstate(h)
|
| 4626 | nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
|
| 4627 | return nr_total_pages;
|
| 4628 | }
|
| 4629 |
|
| 4630 | static int hugetlb_acct_memory(struct hstate *h, long delta)
|
| 4631 | {
|
| 4632 | int ret = -ENOMEM;
|
| 4633 |
|
| 4634 | if (!delta)
|
| 4635 | return 0;
|
| 4636 |
|
| 4637 | spin_lock_irq(&hugetlb_lock);
|
| 4638 | |
| 4639 | * When cpuset is configured, it breaks the strict hugetlb page
|
| 4640 | * reservation as the accounting is done on a global variable. Such
|
| 4641 | * reservation is completely rubbish in the presence of cpuset because
|
| 4642 | * the reservation is not checked against page availability for the
|
| 4643 | * current cpuset. Application can still potentially OOM'ed by kernel
|
| 4644 | * with lack of free htlb page in cpuset that the task is in.
|
| 4645 | * Attempt to enforce strict accounting with cpuset is almost
|
| 4646 | * impossible (or too ugly) because cpuset is too fluid that
|
| 4647 | * task or memory node can be dynamically moved between cpusets.
|
| 4648 | *
|
| 4649 | * The change of semantics for shared hugetlb mapping with cpuset is
|
| 4650 | * undesirable. However, in order to preserve some of the semantics,
|
| 4651 | * we fall back to check against current free page availability as
|
| 4652 | * a best attempt and hopefully to minimize the impact of changing
|
| 4653 | * semantics that cpuset has.
|
| 4654 | *
|
| 4655 | * Apart from cpuset, we also have memory policy mechanism that
|
| 4656 | * also determines from which node the kernel will allocate memory
|
| 4657 | * in a NUMA system. So similar to cpuset, we also should consider
|
| 4658 | * the memory policy of the current task. Similar to the description
|
| 4659 | * above.
|
| 4660 |
|
| 4661 | if (delta > 0) {
|
| 4662 | if (gather_surplus_pages(h, delta) < 0)
|
| 4663 | goto out;
|
| 4664 |
|
| 4665 | if (delta > allowed_mems_nr(h)) {
|
| 4666 | return_unused_surplus_pages(h, delta);
|
| 4667 | goto out;
|
| 4668 | }
|
| 4669 | }
|
| 4670 |
|
| 4671 | ret = 0;
|
| 4672 | if (delta < 0)
|
| 4673 | return_unused_surplus_pages(h, (unsigned long) -delta);
|
| 4674 |
|
| 4675 | out:
|
| 4676 | spin_unlock_irq(&hugetlb_lock);
|
| 4677 | return ret;
|
| 4678 | }
|
| 4679 |
|
| 4680 | static void hugetlb_vm_op_open(struct vm_area_struct *vma)
|
| 4681 | {
|
| 4682 | struct resv_map *resv = vma_resv_map(vma);
|
| 4683 |
|
| 4684 | |
| 4685 | * HPAGE_RESV_OWNER indicates a private mapping.
|
| 4686 | * This new VMA should share its siblings reservation map if present.
|
| 4687 | * The VMA will only ever have a valid reservation map pointer where
|
| 4688 | * it is being copied for another still existing VMA. As that VMA
|
| 4689 | * has a reference to the reservation map it cannot disappear until
|
| 4690 | * after this open call completes. It is therefore safe to take a
|
| 4691 | * new reference here without additional locking.
|
| 4692 |
|
| 4693 | if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
|
| 4694 | resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
|
| 4695 | kref_get(&resv->refs);
|
| 4696 | }
|
| 4697 |
|
| 4698 | |
| 4699 | * vma_lock structure for sharable mappings is vma specific.
|
| 4700 | * Clear old pointer (if copied via vm_area_dup) and allocate
|
| 4701 | * new structure. Before clearing, make sure vma_lock is not
|
| 4702 | * for this vma.
|
| 4703 |
|
| 4704 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 4705 | struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
|
| 4706 |
|
| 4707 | if (vma_lock) {
|
| 4708 | if (vma_lock->vma != vma) {
|
| 4709 | vma->vm_private_data = NULL;
|
| 4710 | hugetlb_vma_lock_alloc(vma);
|
| 4711 | } else {
|
| 4712 | pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__);
|
| 4713 | }
|
| 4714 | } else {
|
| 4715 | hugetlb_vma_lock_alloc(vma);
|
| 4716 | }
|
| 4717 | }
|
| 4718 | }
|
| 4719 |
|
| 4720 | static void hugetlb_vm_op_close(struct vm_area_struct *vma)
|
| 4721 | {
|
| 4722 | struct hstate *h = hstate_vma(vma);
|
| 4723 | struct resv_map *resv;
|
| 4724 | struct hugepage_subpool *spool = subpool_vma(vma);
|
| 4725 | unsigned long reserve, start, end;
|
| 4726 | long gbl_reserve;
|
| 4727 |
|
| 4728 | hugetlb_vma_lock_free(vma);
|
| 4729 |
|
| 4730 | resv = vma_resv_map(vma);
|
| 4731 | if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
|
| 4732 | return;
|
| 4733 |
|
| 4734 | start = vma_hugecache_offset(h, vma, vma->vm_start);
|
| 4735 | end = vma_hugecache_offset(h, vma, vma->vm_end);
|
| 4736 |
|
| 4737 | reserve = (end - start) - region_count(resv, start, end);
|
| 4738 | hugetlb_cgroup_uncharge_counter(resv, start, end);
|
| 4739 | if (reserve) {
|
| 4740 | |
| 4741 | * Decrement reserve counts. The global reserve count may be
|
| 4742 | * adjusted if the subpool has a minimum size.
|
| 4743 |
|
| 4744 | gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
|
| 4745 | hugetlb_acct_memory(h, -gbl_reserve);
|
| 4746 | }
|
| 4747 |
|
| 4748 | kref_put(&resv->refs, resv_map_release);
|
| 4749 | }
|
| 4750 |
|
| 4751 | static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
|
| 4752 | {
|
| 4753 | if (addr & ~(huge_page_mask(hstate_vma(vma))))
|
| 4754 | return -EINVAL;
|
| 4755 | return 0;
|
| 4756 | }
|
| 4757 |
|
| 4758 | void hugetlb_split(struct vm_area_struct *vma, unsigned long addr)
|
| 4759 | {
|
| 4760 | |
| 4761 | * PMD sharing is only possible for PUD_SIZE-aligned address ranges
|
| 4762 | * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this
|
| 4763 | * split, unshare PMDs in the PUD_SIZE interval surrounding addr now.
|
| 4764 | * This function is called in the middle of a VMA split operation, with
|
| 4765 | * MM, VMA and rmap all write-locked to prevent concurrent page table
|
| 4766 | * walks (except hardware and gup_fast()).
|
| 4767 |
|
| 4768 | vma_assert_write_locked(vma);
|
| 4769 | i_mmap_assert_write_locked(vma->vm_file->f_mapping);
|
| 4770 |
|
| 4771 | if (addr & ~PUD_MASK) {
|
| 4772 | unsigned long floor = addr & PUD_MASK;
|
| 4773 | unsigned long ceil = floor + PUD_SIZE;
|
| 4774 |
|
| 4775 | if (floor >= vma->vm_start && ceil <= vma->vm_end) {
|
| 4776 | |
| 4777 | * Locking:
|
| 4778 | * Use take_locks=false here.
|
| 4779 | * The file rmap lock is already held.
|
| 4780 | * The hugetlb VMA lock can't be taken when we already
|
| 4781 | * hold the file rmap lock, and we don't need it because
|
| 4782 | * its purpose is to synchronize against concurrent page
|
| 4783 | * table walks, which are not possible thanks to the
|
| 4784 | * locks held by our caller.
|
| 4785 |
|
| 4786 | hugetlb_unshare_pmds(vma, floor, ceil, false);
|
| 4787 | }
|
| 4788 | }
|
| 4789 | }
|
| 4790 |
|
| 4791 | static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
|
| 4792 | {
|
| 4793 | return huge_page_size(hstate_vma(vma));
|
| 4794 | }
|
| 4795 |
|
| 4796 | |
| 4797 | * We cannot handle pagefaults against hugetlb pages at all. They cause
|
| 4798 | * handle_mm_fault() to try to instantiate regular-sized pages in the
|
| 4799 | * hugepage VMA. do_page_fault() is supposed to trap this, so BUG is we get
|
| 4800 | * this far.
|
| 4801 |
|
| 4802 | static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
|
| 4803 | {
|
| 4804 | BUG();
|
| 4805 | return 0;
|
| 4806 | }
|
| 4807 |
|
| 4808 | #ifdef CONFIG_USERFAULTFD
|
| 4809 | static bool hugetlb_can_userfault(struct vm_area_struct *vma,
|
| 4810 | vm_flags_t vm_flags)
|
| 4811 | {
|
| 4812 | return true;
|
| 4813 | }
|
| 4814 |
|
| 4815 | static const struct vm_uffd_ops hugetlb_uffd_ops = {
|
| 4816 | .can_userfault = hugetlb_can_userfault,
|
| 4817 | };
|
| 4818 | #endif
|
| 4819 |
|
| 4820 | |
| 4821 | * When a new function is introduced to vm_operations_struct and added
|
| 4822 | * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
|
| 4823 | * This is because under System V memory model, mappings created via
|
| 4824 | * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
|
| 4825 | * their original vm_ops are overwritten with shm_vm_ops.
|
| 4826 |
|
| 4827 | const struct vm_operations_struct hugetlb_vm_ops = {
|
| 4828 | .fault = hugetlb_vm_op_fault,
|
| 4829 | .open = hugetlb_vm_op_open,
|
| 4830 | .close = hugetlb_vm_op_close,
|
| 4831 | .may_split = hugetlb_vm_op_split,
|
| 4832 | .pagesize = hugetlb_vm_op_pagesize,
|
| 4833 | #ifdef CONFIG_USERFAULTFD
|
| 4834 | .uffd_ops = &hugetlb_uffd_ops,
|
| 4835 | #endif
|
| 4836 | };
|
| 4837 |
|
| 4838 | static pte_t make_huge_pte(struct vm_area_struct *vma, struct folio *folio,
|
| 4839 | bool try_mkwrite)
|
| 4840 | {
|
| 4841 | pte_t entry = folio_mk_pte(folio, vma->vm_page_prot);
|
| 4842 | unsigned int shift = huge_page_shift(hstate_vma(vma));
|
| 4843 |
|
| 4844 | if (try_mkwrite && (vma->vm_flags & VM_WRITE)) {
|
| 4845 | entry = pte_mkwrite_novma(pte_mkdirty(entry));
|
| 4846 | } else {
|
| 4847 | entry = pte_wrprotect(entry);
|
| 4848 | }
|
| 4849 | entry = pte_mkyoung(entry);
|
| 4850 | entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
|
| 4851 |
|
| 4852 | return entry;
|
| 4853 | }
|
| 4854 |
|
| 4855 | static void set_huge_ptep_writable(struct vm_area_struct *vma,
|
| 4856 | unsigned long address, pte_t *ptep)
|
| 4857 | {
|
| 4858 | pte_t entry;
|
| 4859 |
|
| 4860 | entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(vma->vm_mm, address, ptep)));
|
| 4861 | if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
|
| 4862 | update_mmu_cache(vma, address, ptep);
|
| 4863 | }
|
| 4864 |
|
| 4865 | static void set_huge_ptep_maybe_writable(struct vm_area_struct *vma,
|
| 4866 | unsigned long address, pte_t *ptep)
|
| 4867 | {
|
| 4868 | if (vma->vm_flags & VM_WRITE)
|
| 4869 | set_huge_ptep_writable(vma, address, ptep);
|
| 4870 | }
|
| 4871 |
|
| 4872 | static void
|
| 4873 | hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
|
| 4874 | struct folio *new_folio, pte_t old, unsigned long sz)
|
| 4875 | {
|
| 4876 | pte_t newpte = make_huge_pte(vma, new_folio, true);
|
| 4877 |
|
| 4878 | __folio_mark_uptodate(new_folio);
|
| 4879 | hugetlb_add_new_anon_rmap(new_folio, vma, addr);
|
| 4880 | if (userfaultfd_protected(vma) && huge_pte_uffd(old)) {
|
| 4881 | newpte = huge_pte_mkuffd(newpte);
|
| 4882 |
|
| 4883 | if (userfaultfd_rwp(vma)) {
|
| 4884 | unsigned int shift = huge_page_shift(hstate_vma(vma));
|
| 4885 |
|
| 4886 | newpte = huge_pte_modify(newpte, PAGE_NONE);
|
| 4887 | newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags);
|
| 4888 | }
|
| 4889 | }
|
| 4890 | set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
|
| 4891 | hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
|
| 4892 | folio_set_hugetlb_migratable(new_folio);
|
| 4893 | }
|
| 4894 |
|
| 4895 | int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
|
| 4896 | struct vm_area_struct *dst_vma,
|
| 4897 | struct vm_area_struct *src_vma)
|
| 4898 | {
|
| 4899 | pte_t *src_pte, *dst_pte, entry;
|
| 4900 | struct folio *pte_folio;
|
| 4901 | unsigned long addr;
|
| 4902 | bool cow = vma_is_cow_mapping(src_vma);
|
| 4903 | struct hstate *h = hstate_vma(src_vma);
|
| 4904 | unsigned long sz = huge_page_size(h);
|
| 4905 | unsigned long npages = pages_per_huge_page(h);
|
| 4906 | struct mmu_notifier_range range;
|
| 4907 | unsigned long last_addr_mask;
|
| 4908 | softleaf_t softleaf;
|
| 4909 | int ret = 0;
|
| 4910 |
|
| 4911 | if (cow) {
|
| 4912 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src,
|
| 4913 | src_vma->vm_start,
|
| 4914 | src_vma->vm_end);
|
| 4915 | mmu_notifier_invalidate_range_start(&range);
|
| 4916 | vma_assert_write_locked(src_vma);
|
| 4917 | raw_write_seqcount_begin(&src->write_protect_seq);
|
| 4918 | } else {
|
| 4919 | |
| 4920 | * For shared mappings the vma lock must be held before
|
| 4921 | * calling hugetlb_walk() in the src vma. Otherwise, the
|
| 4922 | * returned ptep could go away if part of a shared pmd and
|
| 4923 | * another thread calls huge_pmd_unshare.
|
| 4924 |
|
| 4925 | hugetlb_vma_lock_read(src_vma);
|
| 4926 | }
|
| 4927 |
|
| 4928 | last_addr_mask = hugetlb_mask_last_page(h);
|
| 4929 | for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
|
| 4930 | spinlock_t *src_ptl, *dst_ptl;
|
| 4931 | src_pte = hugetlb_walk(src_vma, addr, sz);
|
| 4932 | if (!src_pte) {
|
| 4933 | addr |= last_addr_mask;
|
| 4934 | continue;
|
| 4935 | }
|
| 4936 | dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
|
| 4937 | if (!dst_pte) {
|
| 4938 | ret = -ENOMEM;
|
| 4939 | break;
|
| 4940 | }
|
| 4941 |
|
| 4942 | #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
|
| 4943 |
|
| 4944 | if (ptdesc_pmd_is_shared(virt_to_ptdesc(dst_pte))) {
|
| 4945 | addr |= last_addr_mask;
|
| 4946 | continue;
|
| 4947 | }
|
| 4948 | #endif
|
| 4949 |
|
| 4950 | dst_ptl = huge_pte_lock(h, dst, dst_pte);
|
| 4951 | src_ptl = huge_pte_lockptr(h, src, src_pte);
|
| 4952 | spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
|
| 4953 | entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
|
| 4954 | again:
|
| 4955 | if (huge_pte_none(entry)) {
|
| 4956 |
|
| 4957 | goto next;
|
| 4958 | }
|
| 4959 |
|
| 4960 | softleaf = softleaf_from_pte(entry);
|
| 4961 | if (unlikely(softleaf_is_hwpoison(softleaf))) {
|
| 4962 | |
| 4963 | * A hwpoison entry never carries the uffd-wp bit: it is
|
| 4964 | * installed fresh by make_hwpoison_entry() and
|
| 4965 | * hugetlb_change_protection() leaves it untouched, so
|
| 4966 | * there is nothing to clear for the child.
|
| 4967 |
|
| 4968 | set_huge_pte_at(dst, addr, dst_pte, entry, sz);
|
| 4969 | } else if (unlikely(softleaf_is_migration(softleaf))) {
|
| 4970 | bool uffd = pte_swp_uffd(entry);
|
| 4971 |
|
| 4972 | if (!softleaf_is_migration_read(softleaf) && cow) {
|
| 4973 | |
| 4974 | * COW mappings require pages in both
|
| 4975 | * parent and child to be set to read.
|
| 4976 |
|
| 4977 | softleaf = make_readable_migration_entry(
|
| 4978 | swp_offset(softleaf));
|
| 4979 | entry = swp_entry_to_pte(softleaf);
|
| 4980 | if (userfaultfd_protected(src_vma) && uffd)
|
| 4981 | entry = pte_swp_mkuffd(entry);
|
| 4982 | set_huge_pte_at(src, addr, src_pte, entry, sz);
|
| 4983 | }
|
| 4984 | if (!userfaultfd_protected(dst_vma))
|
| 4985 | entry = pte_swp_clear_uffd(entry);
|
| 4986 | set_huge_pte_at(dst, addr, dst_pte, entry, sz);
|
| 4987 | } else if (unlikely(pte_is_marker(entry))) {
|
| 4988 | const pte_marker marker = copy_pte_marker(softleaf, dst_vma);
|
| 4989 |
|
| 4990 | if (marker)
|
| 4991 | set_huge_pte_at(dst, addr, dst_pte,
|
| 4992 | make_pte_marker(marker), sz);
|
| 4993 | } else {
|
| 4994 | entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
|
| 4995 | pte_folio = page_folio(pte_page(entry));
|
| 4996 | folio_get(pte_folio);
|
| 4997 |
|
| 4998 | |
| 4999 | * Failing to duplicate the anon rmap is a rare case
|
| 5000 | * where we see pinned hugetlb pages while they're
|
| 5001 | * prone to COW. We need to do the COW earlier during
|
| 5002 | * fork.
|
| 5003 | *
|
| 5004 | * When pre-allocating the page or copying data, we
|
| 5005 | * need to be without the pgtable locks since we could
|
| 5006 | * sleep during the process.
|
| 5007 |
|
| 5008 | if (!folio_test_anon(pte_folio)) {
|
| 5009 | hugetlb_add_file_rmap(pte_folio);
|
| 5010 | } else if (hugetlb_try_dup_anon_rmap(pte_folio, src_vma)) {
|
| 5011 | pte_t src_pte_old = entry;
|
| 5012 | struct folio *new_folio;
|
| 5013 |
|
| 5014 | spin_unlock(src_ptl);
|
| 5015 | spin_unlock(dst_ptl);
|
| 5016 |
|
| 5017 | new_folio = alloc_hugetlb_folio(dst_vma, addr, false);
|
| 5018 | if (IS_ERR(new_folio)) {
|
| 5019 | folio_put(pte_folio);
|
| 5020 | ret = PTR_ERR(new_folio);
|
| 5021 | break;
|
| 5022 | }
|
| 5023 | ret = copy_user_large_folio(new_folio, pte_folio,
|
| 5024 | addr, dst_vma);
|
| 5025 | folio_put(pte_folio);
|
| 5026 | if (ret) {
|
| 5027 | restore_reserve_on_error(h, dst_vma, addr, new_folio);
|
| 5028 | folio_put(new_folio);
|
| 5029 | break;
|
| 5030 | }
|
| 5031 |
|
| 5032 |
|
| 5033 | dst_ptl = huge_pte_lock(h, dst, dst_pte);
|
| 5034 | src_ptl = huge_pte_lockptr(h, src, src_pte);
|
| 5035 | spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
|
| 5036 | entry = huge_ptep_get(src_vma->vm_mm, addr, src_pte);
|
| 5037 | if (!pte_same(src_pte_old, entry)) {
|
| 5038 | restore_reserve_on_error(h, dst_vma, addr,
|
| 5039 | new_folio);
|
| 5040 | folio_put(new_folio);
|
| 5041 |
|
| 5042 | goto again;
|
| 5043 | }
|
| 5044 | hugetlb_install_folio(dst_vma, dst_pte, addr,
|
| 5045 | new_folio, src_pte_old, sz);
|
| 5046 | goto next;
|
| 5047 | }
|
| 5048 |
|
| 5049 |
|
| 5050 | if (!userfaultfd_protected(dst_vma)) {
|
| 5051 | if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) {
|
| 5052 | entry = huge_pte_modify(entry, dst_vma->vm_page_prot);
|
| 5053 | entry = arch_make_huge_pte(entry, huge_page_shift(h),
|
| 5054 | dst_vma->vm_flags);
|
| 5055 | }
|
| 5056 | entry = huge_pte_clear_uffd(entry);
|
| 5057 | }
|
| 5058 |
|
| 5059 | if (cow) {
|
| 5060 | |
| 5061 | * No need to notify as we are downgrading page
|
| 5062 | * table protection not changing it to point
|
| 5063 | * to a new page.
|
| 5064 | *
|
| 5065 | * See Documentation/mm/mmu_notifier.rst
|
| 5066 |
|
| 5067 | huge_ptep_set_wrprotect(src, addr, src_pte);
|
| 5068 | entry = huge_pte_wrprotect(entry);
|
| 5069 | }
|
| 5070 |
|
| 5071 | set_huge_pte_at(dst, addr, dst_pte, entry, sz);
|
| 5072 | hugetlb_count_add(npages, dst);
|
| 5073 | }
|
| 5074 |
|
| 5075 | next:
|
| 5076 | spin_unlock(src_ptl);
|
| 5077 | spin_unlock(dst_ptl);
|
| 5078 | }
|
| 5079 |
|
| 5080 | if (cow) {
|
| 5081 | raw_write_seqcount_end(&src->write_protect_seq);
|
| 5082 | mmu_notifier_invalidate_range_end(&range);
|
| 5083 | } else {
|
| 5084 | hugetlb_vma_unlock_read(src_vma);
|
| 5085 | }
|
| 5086 |
|
| 5087 | return ret;
|
| 5088 | }
|
| 5089 |
|
| 5090 | static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
|
| 5091 | unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte,
|
| 5092 | unsigned long sz)
|
| 5093 | {
|
| 5094 | bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
|
| 5095 | struct hstate *h = hstate_vma(vma);
|
| 5096 | struct mm_struct *mm = vma->vm_mm;
|
| 5097 | spinlock_t *src_ptl, *dst_ptl;
|
| 5098 | pte_t pte;
|
| 5099 |
|
| 5100 | dst_ptl = huge_pte_lock(h, mm, dst_pte);
|
| 5101 | src_ptl = huge_pte_lockptr(h, mm, src_pte);
|
| 5102 |
|
| 5103 | |
| 5104 | * We don't have to worry about the ordering of src and dst ptlocks
|
| 5105 | * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
|
| 5106 |
|
| 5107 | if (src_ptl != dst_ptl)
|
| 5108 | spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
|
| 5109 |
|
| 5110 | pte = huge_ptep_get_and_clear(mm, old_addr, src_pte, sz);
|
| 5111 |
|
| 5112 | if (need_clear_uffd_wp && pte_is_uffd_wp_marker(pte)) {
|
| 5113 | huge_pte_clear(mm, new_addr, dst_pte, sz);
|
| 5114 | } else {
|
| 5115 | if (need_clear_uffd_wp) {
|
| 5116 | if (pte_present(pte)) {
|
| 5117 | |
| 5118 | * See __copy_present_ptes(): normalise the RWP
|
| 5119 | * marker so the destination starts accessible
|
| 5120 | * instead of taking a numa-hinting fault on
|
| 5121 | * first access. Only the marker (protnone + uffd)
|
| 5122 | * needs it; leave other present PTEs untouched.
|
| 5123 |
|
| 5124 | if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
|
| 5125 | pte = huge_pte_modify(pte, vma->vm_page_prot);
|
| 5126 | pte = arch_make_huge_pte(pte, huge_page_shift(h),
|
| 5127 | vma->vm_flags);
|
| 5128 | }
|
| 5129 | pte = huge_pte_clear_uffd(pte);
|
| 5130 | } else {
|
| 5131 | pte = pte_swp_clear_uffd(pte);
|
| 5132 | }
|
| 5133 | }
|
| 5134 | set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
|
| 5135 | }
|
| 5136 |
|
| 5137 | if (src_ptl != dst_ptl)
|
| 5138 | spin_unlock(src_ptl);
|
| 5139 | spin_unlock(dst_ptl);
|
| 5140 | }
|
| 5141 |
|
| 5142 | int move_hugetlb_page_tables(struct vm_area_struct *vma,
|
| 5143 | struct vm_area_struct *new_vma,
|
| 5144 | unsigned long old_addr, unsigned long new_addr,
|
| 5145 | unsigned long len)
|
| 5146 | {
|
| 5147 | struct hstate *h = hstate_vma(vma);
|
| 5148 | struct address_space *mapping = vma->vm_file->f_mapping;
|
| 5149 | unsigned long sz = huge_page_size(h);
|
| 5150 | struct mm_struct *mm = vma->vm_mm;
|
| 5151 | unsigned long old_end = old_addr + len;
|
| 5152 | unsigned long last_addr_mask;
|
| 5153 | pte_t *src_pte, *dst_pte;
|
| 5154 | struct mmu_notifier_range range;
|
| 5155 | struct mmu_gather tlb;
|
| 5156 |
|
| 5157 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr,
|
| 5158 | old_end);
|
| 5159 | adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
|
| 5160 | |
| 5161 | * In case of shared PMDs, we should cover the maximum possible
|
| 5162 | * range.
|
| 5163 |
|
| 5164 | flush_cache_range(vma, range.start, range.end);
|
| 5165 | tlb_gather_mmu_vma(&tlb, vma);
|
| 5166 |
|
| 5167 | mmu_notifier_invalidate_range_start(&range);
|
| 5168 | last_addr_mask = hugetlb_mask_last_page(h);
|
| 5169 |
|
| 5170 | hugetlb_vma_lock_write(vma);
|
| 5171 | i_mmap_lock_write(mapping);
|
| 5172 | for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
|
| 5173 | const unsigned long offset_to_last_entry =
|
| 5174 | (old_addr | last_addr_mask) - old_addr;
|
| 5175 |
|
| 5176 | src_pte = hugetlb_walk(vma, old_addr, sz);
|
| 5177 | if (!src_pte) {
|
| 5178 | old_addr += offset_to_last_entry;
|
| 5179 | new_addr += offset_to_last_entry;
|
| 5180 | continue;
|
| 5181 | }
|
| 5182 | if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte)))
|
| 5183 | continue;
|
| 5184 |
|
| 5185 | if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) {
|
| 5186 | old_addr += offset_to_last_entry;
|
| 5187 | new_addr += offset_to_last_entry;
|
| 5188 | continue;
|
| 5189 | }
|
| 5190 |
|
| 5191 | dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
|
| 5192 | if (!dst_pte)
|
| 5193 | break;
|
| 5194 |
|
| 5195 | move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
|
| 5196 | tlb_remove_huge_tlb_entry(h, &tlb, src_pte, old_addr);
|
| 5197 | }
|
| 5198 |
|
| 5199 | tlb_flush_mmu_tlbonly(&tlb);
|
| 5200 | huge_pmd_unshare_flush(&tlb, vma);
|
| 5201 |
|
| 5202 | mmu_notifier_invalidate_range_end(&range);
|
| 5203 | i_mmap_unlock_write(mapping);
|
| 5204 | hugetlb_vma_unlock_write(vma);
|
| 5205 | tlb_finish_mmu(&tlb);
|
| 5206 |
|
| 5207 | return len + old_addr - old_end;
|
| 5208 | }
|
| 5209 |
|
| 5210 | void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
|
| 5211 | unsigned long start, unsigned long end,
|
| 5212 | struct folio *folio, zap_flags_t zap_flags)
|
| 5213 | {
|
| 5214 | struct mm_struct *mm = vma->vm_mm;
|
| 5215 | const bool folio_provided = !!folio;
|
| 5216 | unsigned long address;
|
| 5217 | pte_t *ptep;
|
| 5218 | pte_t pte;
|
| 5219 | spinlock_t *ptl;
|
| 5220 | struct hstate *h = hstate_vma(vma);
|
| 5221 | unsigned long sz = huge_page_size(h);
|
| 5222 | bool adjust_reservation;
|
| 5223 | unsigned long last_addr_mask;
|
| 5224 |
|
| 5225 | i_mmap_assert_write_locked(vma->vm_file->f_mapping);
|
| 5226 | WARN_ON(!is_vm_hugetlb_page(vma));
|
| 5227 | BUG_ON(start & ~huge_page_mask(h));
|
| 5228 | BUG_ON(end & ~huge_page_mask(h));
|
| 5229 |
|
| 5230 | |
| 5231 | * This is a hugetlb vma, all the pte entries should point
|
| 5232 | * to huge page.
|
| 5233 |
|
| 5234 | tlb_change_page_size(tlb, sz);
|
| 5235 | tlb_start_vma(tlb, vma);
|
| 5236 |
|
| 5237 | last_addr_mask = hugetlb_mask_last_page(h);
|
| 5238 | address = start;
|
| 5239 | for (; address < end; address += sz) {
|
| 5240 | ptep = hugetlb_walk(vma, address, sz);
|
| 5241 | if (!ptep) {
|
| 5242 | address |= last_addr_mask;
|
| 5243 | continue;
|
| 5244 | }
|
| 5245 |
|
| 5246 | ptl = huge_pte_lock(h, mm, ptep);
|
| 5247 | if (huge_pmd_unshare(tlb, vma, address, ptep)) {
|
| 5248 | spin_unlock(ptl);
|
| 5249 | address |= last_addr_mask;
|
| 5250 | continue;
|
| 5251 | }
|
| 5252 |
|
| 5253 | pte = huge_ptep_get(mm, address, ptep);
|
| 5254 | if (huge_pte_none(pte)) {
|
| 5255 | spin_unlock(ptl);
|
| 5256 | continue;
|
| 5257 | }
|
| 5258 |
|
| 5259 | |
| 5260 | * Migrating hugepage or HWPoisoned hugepage is already
|
| 5261 | * unmapped and its refcount is dropped, so just clear pte here.
|
| 5262 |
|
| 5263 | if (unlikely(!pte_present(pte))) {
|
| 5264 | |
| 5265 | * If the pte was wr-protected by uffd-wp in any of the
|
| 5266 | * swap forms, meanwhile the caller does not want to
|
| 5267 | * drop the uffd-wp bit in this zap, then replace the
|
| 5268 | * pte with a marker.
|
| 5269 |
|
| 5270 | if (pte_swp_uffd_any(pte) &&
|
| 5271 | !(zap_flags & ZAP_FLAG_DROP_MARKER))
|
| 5272 | set_huge_pte_at(mm, address, ptep,
|
| 5273 | make_pte_marker(PTE_MARKER_UFFD_WP),
|
| 5274 | sz);
|
| 5275 | else
|
| 5276 | huge_pte_clear(mm, address, ptep, sz);
|
| 5277 | spin_unlock(ptl);
|
| 5278 | continue;
|
| 5279 | }
|
| 5280 |
|
| 5281 | |
| 5282 | * If a folio is supplied, it is because a specific
|
| 5283 | * folio is being unmapped, not a range. Ensure the folio we
|
| 5284 | * are about to unmap is the actual folio of interest.
|
| 5285 |
|
| 5286 | if (folio_provided) {
|
| 5287 | if (folio != page_folio(pte_page(pte))) {
|
| 5288 | spin_unlock(ptl);
|
| 5289 | continue;
|
| 5290 | }
|
| 5291 | |
| 5292 | * Mark the VMA as having unmapped its page so that
|
| 5293 | * future faults in this VMA will fail rather than
|
| 5294 | * looking like data was lost
|
| 5295 |
|
| 5296 | set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
|
| 5297 | } else {
|
| 5298 | folio = page_folio(pte_page(pte));
|
| 5299 | }
|
| 5300 |
|
| 5301 | pte = huge_ptep_get_and_clear(mm, address, ptep, sz);
|
| 5302 | tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
|
| 5303 | if (huge_pte_dirty(pte))
|
| 5304 | folio_mark_dirty(folio);
|
| 5305 |
|
| 5306 | if (huge_pte_uffd(pte) &&
|
| 5307 | !(zap_flags & ZAP_FLAG_DROP_MARKER))
|
| 5308 | set_huge_pte_at(mm, address, ptep,
|
| 5309 | make_pte_marker(PTE_MARKER_UFFD_WP),
|
| 5310 | sz);
|
| 5311 | hugetlb_count_sub(pages_per_huge_page(h), mm);
|
| 5312 | hugetlb_remove_rmap(folio);
|
| 5313 | spin_unlock(ptl);
|
| 5314 |
|
| 5315 | |
| 5316 | * Restore the reservation for anonymous page, otherwise the
|
| 5317 | * backing page could be stolen by someone. Restore only on the
|
| 5318 | * last unmap, otherwise the owner could empty its resv map
|
| 5319 | * while the folio is still mapped by a child. Note that holding
|
| 5320 | * i_mmap_lock_write is needed to check the number of mappings.
|
| 5321 | * If there we are freeing a surplus, do not set the restore
|
| 5322 | * reservation bit.
|
| 5323 |
|
| 5324 | adjust_reservation = false;
|
| 5325 |
|
| 5326 | spin_lock_irq(&hugetlb_lock);
|
| 5327 | if (!h->surplus_huge_pages && __vma_private_lock(vma) &&
|
| 5328 | !folio_mapped(folio) && folio_test_anon(folio)) {
|
| 5329 | folio_set_hugetlb_restore_reserve(folio);
|
| 5330 |
|
| 5331 | adjust_reservation = true;
|
| 5332 | }
|
| 5333 | spin_unlock_irq(&hugetlb_lock);
|
| 5334 |
|
| 5335 | |
| 5336 | * Adjust the reservation for the region that will have the
|
| 5337 | * reserve restored. Keep in mind that vma_needs_reservation() changes
|
| 5338 | * resv->adds_in_progress if it succeeds. If this is not done,
|
| 5339 | * do_exit() will not see it, and will keep the reservation
|
| 5340 | * forever.
|
| 5341 |
|
| 5342 | if (adjust_reservation) {
|
| 5343 | int rc = vma_needs_reservation(h, vma, address);
|
| 5344 |
|
| 5345 | if (rc < 0)
|
| 5346 | |
| 5347 | * to allocate a file_region struct. Clear
|
| 5348 | * hugetlb_restore_reserve so that global reserve
|
| 5349 | * count will not be incremented by free_huge_folio.
|
| 5350 | * Act as if we consumed the reservation.
|
| 5351 |
|
| 5352 | folio_clear_hugetlb_restore_reserve(folio);
|
| 5353 | else if (rc)
|
| 5354 | vma_add_reservation(h, vma, address);
|
| 5355 | }
|
| 5356 |
|
| 5357 | tlb_remove_page_size(tlb, folio_page(folio, 0),
|
| 5358 | folio_size(folio));
|
| 5359 | |
| 5360 | * If we were instructed to unmap a specific folio, we're done.
|
| 5361 |
|
| 5362 | if (folio_provided)
|
| 5363 | break;
|
| 5364 | }
|
| 5365 | tlb_end_vma(tlb, vma);
|
| 5366 |
|
| 5367 | huge_pmd_unshare_flush(tlb, vma);
|
| 5368 | }
|
| 5369 |
|
| 5370 | void __hugetlb_zap_begin(struct vm_area_struct *vma,
|
| 5371 | unsigned long *start, unsigned long *end)
|
| 5372 | {
|
| 5373 | if (!vma->vm_file)
|
| 5374 | return;
|
| 5375 |
|
| 5376 | adjust_range_if_pmd_sharing_possible(vma, start, end);
|
| 5377 | hugetlb_vma_lock_write(vma);
|
| 5378 | if (vma->vm_file)
|
| 5379 | i_mmap_lock_write(vma->vm_file->f_mapping);
|
| 5380 | }
|
| 5381 |
|
| 5382 | void __hugetlb_zap_end(struct vm_area_struct *vma,
|
| 5383 | struct zap_details *details)
|
| 5384 | {
|
| 5385 | zap_flags_t zap_flags = details ? details->zap_flags : 0;
|
| 5386 |
|
| 5387 | if (!vma->vm_file)
|
| 5388 | return;
|
| 5389 |
|
| 5390 | if (zap_flags & ZAP_FLAG_UNMAP) {
|
| 5391 | |
| 5392 | * Unlock and free the vma lock before releasing i_mmap_rwsem.
|
| 5393 | * When the vma_lock is freed, this makes the vma ineligible
|
| 5394 | * for pmd sharing. And, i_mmap_rwsem is required to set up
|
| 5395 | * pmd sharing. This is important as page tables for this
|
| 5396 | * unmapped range will be asynchrously deleted. If the page
|
| 5397 | * tables are shared, there will be issues when accessed by
|
| 5398 | * someone else.
|
| 5399 |
|
| 5400 | __hugetlb_vma_unlock_write_free(vma);
|
| 5401 | } else {
|
| 5402 | hugetlb_vma_unlock_write(vma);
|
| 5403 | }
|
| 5404 |
|
| 5405 | if (vma->vm_file)
|
| 5406 | i_mmap_unlock_write(vma->vm_file->f_mapping);
|
| 5407 | }
|
| 5408 |
|
| 5409 | void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
|
| 5410 | unsigned long end, struct folio *folio,
|
| 5411 | zap_flags_t zap_flags)
|
| 5412 | {
|
| 5413 | struct mmu_notifier_range range;
|
| 5414 | struct mmu_gather tlb;
|
| 5415 |
|
| 5416 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
|
| 5417 | start, end);
|
| 5418 | adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
|
| 5419 | mmu_notifier_invalidate_range_start(&range);
|
| 5420 | tlb_gather_mmu(&tlb, vma->vm_mm);
|
| 5421 |
|
| 5422 | __unmap_hugepage_range(&tlb, vma, start, end,
|
| 5423 | folio, zap_flags);
|
| 5424 |
|
| 5425 | mmu_notifier_invalidate_range_end(&range);
|
| 5426 | tlb_finish_mmu(&tlb);
|
| 5427 | }
|
| 5428 |
|
| 5429 | |
| 5430 | * This is called when the original mapper is failing to COW a MAP_PRIVATE
|
| 5431 | * mapping it owns the reserve page for. The intention is to unmap the page
|
| 5432 | * from other VMAs and let the children be SIGKILLed if they are faulting the
|
| 5433 | * same region.
|
| 5434 |
|
| 5435 | static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
|
| 5436 | struct folio *folio, unsigned long address)
|
| 5437 | {
|
| 5438 | struct hstate *h = hstate_vma(vma);
|
| 5439 | struct vm_area_struct *iter_vma;
|
| 5440 | struct address_space *mapping;
|
| 5441 | pgoff_t pgoff;
|
| 5442 |
|
| 5443 | |
| 5444 | * vm_pgoff is in PAGE_SIZE units, hence the different calculation
|
| 5445 | * from page cache lookup which is in HPAGE_SIZE units.
|
| 5446 |
|
| 5447 | address = address & huge_page_mask(h);
|
| 5448 | pgoff = linear_page_index(vma, address);
|
| 5449 | mapping = vma->vm_file->f_mapping;
|
| 5450 |
|
| 5451 | |
| 5452 | * Take the mapping lock for the duration of the table walk. As
|
| 5453 | * this mapping should be shared between all the VMAs,
|
| 5454 | * __unmap_hugepage_range() is called as the lock is already held
|
| 5455 |
|
| 5456 | i_mmap_lock_write(mapping);
|
| 5457 | mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) {
|
| 5458 |
|
| 5459 | if (iter_vma == vma)
|
| 5460 | continue;
|
| 5461 |
|
| 5462 | |
| 5463 | * Shared VMAs have their own reserves and do not affect
|
| 5464 | * MAP_PRIVATE accounting but it is possible that a shared
|
| 5465 | * VMA is using the same page so check and skip such VMAs.
|
| 5466 |
|
| 5467 | if (iter_vma->vm_flags & VM_MAYSHARE)
|
| 5468 | continue;
|
| 5469 |
|
| 5470 | |
| 5471 | * Unmap the page from other VMAs without their own reserves.
|
| 5472 | * They get marked to be SIGKILLed if they fault in these
|
| 5473 | * areas. This is because a future no-page fault on this VMA
|
| 5474 | * could insert a zeroed page instead of the data existing
|
| 5475 | * from the time of fork. This would look like data corruption
|
| 5476 |
|
| 5477 | if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
|
| 5478 | unmap_hugepage_range(iter_vma, address,
|
| 5479 | address + huge_page_size(h),
|
| 5480 | folio, 0);
|
| 5481 | }
|
| 5482 | i_mmap_unlock_write(mapping);
|
| 5483 | }
|
| 5484 |
|
| 5485 | |
| 5486 | * hugetlb_wp() should be called with page lock of the original hugepage held.
|
| 5487 | * Called with hugetlb_fault_mutex_table held and pte_page locked so we
|
| 5488 | * cannot race with other handlers or page migration.
|
| 5489 | * Keep the pte_same checks anyway to make transition from the mutex easier.
|
| 5490 |
|
| 5491 | static vm_fault_t hugetlb_wp(struct vm_fault *vmf)
|
| 5492 | {
|
| 5493 | struct vm_area_struct *vma = vmf->vma;
|
| 5494 | struct mm_struct *mm = vma->vm_mm;
|
| 5495 | const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
|
| 5496 | pte_t pte = huge_ptep_get(mm, vmf->address, vmf->pte);
|
| 5497 | struct hstate *h = hstate_vma(vma);
|
| 5498 | struct folio *old_folio;
|
| 5499 | struct folio *new_folio;
|
| 5500 | bool cow_from_owner = 0;
|
| 5501 | vm_fault_t ret = 0;
|
| 5502 | struct mmu_notifier_range range;
|
| 5503 |
|
| 5504 | |
| 5505 | * Never handle CoW for uffd-wp protected pages. It should be only
|
| 5506 | * handled when the uffd-wp protection is removed.
|
| 5507 | *
|
| 5508 | * Note that only the CoW optimization path (in hugetlb_no_page())
|
| 5509 | * can trigger this, because hugetlb_fault() will always resolve
|
| 5510 | * uffd-wp bit first.
|
| 5511 |
|
| 5512 | if (!unshare && huge_pte_uffd(pte))
|
| 5513 | return 0;
|
| 5514 |
|
| 5515 |
|
| 5516 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 5517 | set_huge_ptep_writable(vma, vmf->address, vmf->pte);
|
| 5518 | return 0;
|
| 5519 | }
|
| 5520 |
|
| 5521 | old_folio = page_folio(pte_page(pte));
|
| 5522 |
|
| 5523 | delayacct_wpcopy_start();
|
| 5524 |
|
| 5525 | retry_avoidcopy:
|
| 5526 | |
| 5527 | * If no-one else is actually using this page, we're the exclusive
|
| 5528 | * owner and can reuse this page.
|
| 5529 | *
|
| 5530 | * Note that we don't rely on the (safer) folio refcount here, because
|
| 5531 | * copying the hugetlb folio when there are unexpected (temporary)
|
| 5532 | * folio references could harm simple fork()+exit() users when
|
| 5533 | * we run out of free hugetlb folios: we would have to kill processes
|
| 5534 | * in scenarios that used to work. As a side effect, there can still
|
| 5535 | * be leaks between processes, for example, with FOLL_GET users.
|
| 5536 |
|
| 5537 | if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) {
|
| 5538 | if (!PageAnonExclusive(&old_folio->page)) {
|
| 5539 | folio_move_anon_rmap(old_folio, vma);
|
| 5540 | SetPageAnonExclusive(&old_folio->page);
|
| 5541 | }
|
| 5542 | if (likely(!unshare))
|
| 5543 | set_huge_ptep_maybe_writable(vma, vmf->address,
|
| 5544 | vmf->pte);
|
| 5545 |
|
| 5546 | delayacct_wpcopy_end();
|
| 5547 | return 0;
|
| 5548 | }
|
| 5549 | VM_BUG_ON_PAGE(folio_test_anon(old_folio) &&
|
| 5550 | PageAnonExclusive(&old_folio->page), &old_folio->page);
|
| 5551 |
|
| 5552 | |
| 5553 | * If the process that created a MAP_PRIVATE mapping is about to perform
|
| 5554 | * a COW due to a shared page count, attempt to satisfy the allocation
|
| 5555 | * without using the existing reserves.
|
| 5556 | * In order to determine where this is a COW on a MAP_PRIVATE mapping it
|
| 5557 | * is enough to check whether the old_folio is anonymous. This means that
|
| 5558 | * the reserve for this address was consumed. If reserves were used, a
|
| 5559 | * partial faulted mapping at the fime of fork() could consume its reserves
|
| 5560 | * on COW instead of the full address range.
|
| 5561 |
|
| 5562 | if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
|
| 5563 | folio_test_anon(old_folio))
|
| 5564 | cow_from_owner = true;
|
| 5565 |
|
| 5566 | folio_get(old_folio);
|
| 5567 |
|
| 5568 | |
| 5569 | * Drop page table lock as buddy allocator may be called. It will
|
| 5570 | * be acquired again before returning to the caller, as expected.
|
| 5571 |
|
| 5572 | spin_unlock(vmf->ptl);
|
| 5573 | new_folio = alloc_hugetlb_folio(vma, vmf->address, cow_from_owner);
|
| 5574 |
|
| 5575 | if (IS_ERR(new_folio)) {
|
| 5576 | |
| 5577 | * If a process owning a MAP_PRIVATE mapping fails to COW,
|
| 5578 | * it is due to references held by a child and an insufficient
|
| 5579 | * huge page pool. To guarantee the original mappers
|
| 5580 | * reliability, unmap the page from child processes. The child
|
| 5581 | * may get SIGKILLed if it later faults.
|
| 5582 |
|
| 5583 | if (cow_from_owner) {
|
| 5584 | struct address_space *mapping = vma->vm_file->f_mapping;
|
| 5585 | pgoff_t idx;
|
| 5586 | u32 hash;
|
| 5587 |
|
| 5588 | folio_put(old_folio);
|
| 5589 | |
| 5590 | * Drop hugetlb_fault_mutex and vma_lock before
|
| 5591 | * unmapping. unmapping needs to hold vma_lock
|
| 5592 | * in write mode. Dropping vma_lock in read mode
|
| 5593 | * here is OK as COW mappings do not interact with
|
| 5594 | * PMD sharing.
|
| 5595 | *
|
| 5596 | * Reacquire both after unmap operation.
|
| 5597 |
|
| 5598 | idx = vma_hugecache_offset(h, vma, vmf->address);
|
| 5599 | hash = hugetlb_fault_mutex_hash(mapping, idx);
|
| 5600 | hugetlb_vma_unlock_read(vma);
|
| 5601 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 5602 |
|
| 5603 | unmap_ref_private(mm, vma, old_folio, vmf->address);
|
| 5604 |
|
| 5605 | mutex_lock(&hugetlb_fault_mutex_table[hash]);
|
| 5606 | hugetlb_vma_lock_read(vma);
|
| 5607 | spin_lock(vmf->ptl);
|
| 5608 | vmf->pte = hugetlb_walk(vma, vmf->address,
|
| 5609 | huge_page_size(h));
|
| 5610 | if (likely(vmf->pte &&
|
| 5611 | pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), pte)))
|
| 5612 | goto retry_avoidcopy;
|
| 5613 | |
| 5614 | * race occurs while re-acquiring page table
|
| 5615 | * lock, and our job is done.
|
| 5616 |
|
| 5617 | delayacct_wpcopy_end();
|
| 5618 | return 0;
|
| 5619 | }
|
| 5620 |
|
| 5621 | ret = vmf_error(PTR_ERR(new_folio));
|
| 5622 | goto out_release_old;
|
| 5623 | }
|
| 5624 |
|
| 5625 | |
| 5626 | * When the original hugepage is shared one, it does not have
|
| 5627 | * anon_vma prepared.
|
| 5628 |
|
| 5629 | ret = __vmf_anon_prepare(vmf);
|
| 5630 | if (unlikely(ret))
|
| 5631 | goto out_release_all;
|
| 5632 |
|
| 5633 | if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) {
|
| 5634 | ret = VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(hstate_index(h));
|
| 5635 | goto out_release_all;
|
| 5636 | }
|
| 5637 | __folio_mark_uptodate(new_folio);
|
| 5638 |
|
| 5639 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address,
|
| 5640 | vmf->address + huge_page_size(h));
|
| 5641 | mmu_notifier_invalidate_range_start(&range);
|
| 5642 |
|
| 5643 | |
| 5644 | * Retake the page table lock to check for racing updates
|
| 5645 | * before the page tables are altered
|
| 5646 |
|
| 5647 | spin_lock(vmf->ptl);
|
| 5648 | vmf->pte = hugetlb_walk(vma, vmf->address, huge_page_size(h));
|
| 5649 | if (likely(vmf->pte && pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), pte))) {
|
| 5650 | pte_t newpte = make_huge_pte(vma, new_folio, !unshare);
|
| 5651 |
|
| 5652 |
|
| 5653 | huge_ptep_clear_flush(vma, vmf->address, vmf->pte);
|
| 5654 | hugetlb_remove_rmap(old_folio);
|
| 5655 | hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address);
|
| 5656 | if (huge_pte_uffd(pte))
|
| 5657 | newpte = huge_pte_mkuffd(newpte);
|
| 5658 | set_huge_pte_at(mm, vmf->address, vmf->pte, newpte,
|
| 5659 | huge_page_size(h));
|
| 5660 | folio_set_hugetlb_migratable(new_folio);
|
| 5661 |
|
| 5662 | new_folio = old_folio;
|
| 5663 | }
|
| 5664 | spin_unlock(vmf->ptl);
|
| 5665 | mmu_notifier_invalidate_range_end(&range);
|
| 5666 | out_release_all:
|
| 5667 | |
| 5668 | * No restore in case of successful pagetable update (Break COW or
|
| 5669 | * unshare)
|
| 5670 |
|
| 5671 | if (new_folio != old_folio)
|
| 5672 | restore_reserve_on_error(h, vma, vmf->address, new_folio);
|
| 5673 | folio_put(new_folio);
|
| 5674 | out_release_old:
|
| 5675 | folio_put(old_folio);
|
| 5676 |
|
| 5677 | spin_lock(vmf->ptl);
|
| 5678 |
|
| 5679 | delayacct_wpcopy_end();
|
| 5680 | return ret;
|
| 5681 | }
|
| 5682 |
|
| 5683 | |
| 5684 | * Return whether there is a pagecache page to back given address within VMA.
|
| 5685 |
|
| 5686 | bool hugetlbfs_pagecache_present(struct hstate *h,
|
| 5687 | struct vm_area_struct *vma, unsigned long address)
|
| 5688 | {
|
| 5689 | struct address_space *mapping = vma->vm_file->f_mapping;
|
| 5690 | pgoff_t idx = linear_page_index(vma, address);
|
| 5691 | struct folio *folio;
|
| 5692 |
|
| 5693 | folio = filemap_get_folio(mapping, idx);
|
| 5694 | if (IS_ERR(folio))
|
| 5695 | return false;
|
| 5696 | folio_put(folio);
|
| 5697 | return true;
|
| 5698 | }
|
| 5699 |
|
| 5700 | int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
|
| 5701 | pgoff_t idx)
|
| 5702 | {
|
| 5703 | struct inode *inode = mapping->host;
|
| 5704 | struct hstate *h = hstate_inode(inode);
|
| 5705 | int err;
|
| 5706 |
|
| 5707 | idx <<= huge_page_order(h);
|
| 5708 | __folio_set_locked(folio);
|
| 5709 | err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL);
|
| 5710 |
|
| 5711 | if (unlikely(err)) {
|
| 5712 | __folio_clear_locked(folio);
|
| 5713 | return err;
|
| 5714 | }
|
| 5715 | folio_clear_hugetlb_restore_reserve(folio);
|
| 5716 |
|
| 5717 | |
| 5718 | * mark folio dirty so that it will not be removed from cache/file
|
| 5719 | * by non-hugetlbfs specific code paths.
|
| 5720 |
|
| 5721 | folio_mark_dirty(folio);
|
| 5722 |
|
| 5723 | spin_lock(&inode->i_lock);
|
| 5724 | inode->i_blocks += blocks_per_huge_page(h);
|
| 5725 | spin_unlock(&inode->i_lock);
|
| 5726 | return 0;
|
| 5727 | }
|
| 5728 |
|
| 5729 | static inline vm_fault_t hugetlb_handle_userfault(struct vm_fault *vmf,
|
| 5730 | struct address_space *mapping,
|
| 5731 | unsigned long reason)
|
| 5732 | {
|
| 5733 | u32 hash;
|
| 5734 |
|
| 5735 | |
| 5736 | * vma_lock and hugetlb_fault_mutex must be dropped before handling
|
| 5737 | * userfault. Also mmap_lock could be dropped due to handling
|
| 5738 | * userfault, any vma operation should be careful from here.
|
| 5739 |
|
| 5740 | hugetlb_vma_unlock_read(vmf->vma);
|
| 5741 | hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
|
| 5742 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 5743 | return handle_userfault(vmf, reason);
|
| 5744 | }
|
| 5745 |
|
| 5746 | |
| 5747 | * Recheck pte with pgtable lock. Returns true if pte didn't change, or
|
| 5748 | * false if pte changed or is changing.
|
| 5749 |
|
| 5750 | static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, unsigned long addr,
|
| 5751 | pte_t *ptep, pte_t old_pte)
|
| 5752 | {
|
| 5753 | spinlock_t *ptl;
|
| 5754 | bool same;
|
| 5755 |
|
| 5756 | ptl = huge_pte_lock(h, mm, ptep);
|
| 5757 | same = pte_same(huge_ptep_get(mm, addr, ptep), old_pte);
|
| 5758 | spin_unlock(ptl);
|
| 5759 |
|
| 5760 | return same;
|
| 5761 | }
|
| 5762 |
|
| 5763 | static vm_fault_t hugetlb_no_page(struct address_space *mapping,
|
| 5764 | struct vm_fault *vmf)
|
| 5765 | {
|
| 5766 | u32 hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
|
| 5767 | bool new_folio, new_anon_folio = false;
|
| 5768 | struct vm_area_struct *vma = vmf->vma;
|
| 5769 | struct mm_struct *mm = vma->vm_mm;
|
| 5770 | struct hstate *h = hstate_vma(vma);
|
| 5771 | vm_fault_t ret = VM_FAULT_SIGBUS;
|
| 5772 | bool folio_locked = true;
|
| 5773 | struct folio *folio;
|
| 5774 | unsigned long size;
|
| 5775 | pte_t new_pte;
|
| 5776 |
|
| 5777 | |
| 5778 | * Currently, we are forced to kill the process in the event the
|
| 5779 | * original mapper has unmapped pages from the child due to a failed
|
| 5780 | * COW/unsharing. Warn that such a situation has occurred as it may not
|
| 5781 | * be obvious.
|
| 5782 |
|
| 5783 | if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
|
| 5784 | pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
|
| 5785 | current->pid);
|
| 5786 | goto out;
|
| 5787 | }
|
| 5788 |
|
| 5789 | |
| 5790 | * Use page lock to guard against racing truncation
|
| 5791 | * before we get page_table_lock.
|
| 5792 |
|
| 5793 | new_folio = false;
|
| 5794 | folio = filemap_lock_hugetlb_folio(h, mapping, vmf->pgoff);
|
| 5795 | if (IS_ERR(folio)) {
|
| 5796 | size = i_size_read(mapping->host) >> huge_page_shift(h);
|
| 5797 | if (vmf->pgoff >= size)
|
| 5798 | goto out;
|
| 5799 |
|
| 5800 | if (userfaultfd_missing(vma)) {
|
| 5801 | |
| 5802 | * Since hugetlb_no_page() was examining pte
|
| 5803 | * without pgtable lock, we need to re-test under
|
| 5804 | * lock because the pte may not be stable and could
|
| 5805 | * have changed from under us. Try to detect
|
| 5806 | * either changed or during-changing ptes and retry
|
| 5807 | * properly when needed.
|
| 5808 | *
|
| 5809 | * Note that userfaultfd is actually fine with
|
| 5810 | * false positives (e.g. caused by pte changed),
|
| 5811 | * but not wrong logical events (e.g. caused by
|
| 5812 | * reading a pte during changing). The latter can
|
| 5813 | * confuse the userspace, so the strictness is very
|
| 5814 | * much preferred. E.g., MISSING event should
|
| 5815 | * never happen on the page after UFFDIO_COPY has
|
| 5816 | * correctly installed the page and returned.
|
| 5817 |
|
| 5818 | if (!hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte)) {
|
| 5819 | ret = 0;
|
| 5820 | goto out;
|
| 5821 | }
|
| 5822 |
|
| 5823 | return hugetlb_handle_userfault(vmf, mapping,
|
| 5824 | VM_UFFD_MISSING);
|
| 5825 | }
|
| 5826 |
|
| 5827 | if (!(vma->vm_flags & VM_MAYSHARE)) {
|
| 5828 | ret = __vmf_anon_prepare(vmf);
|
| 5829 | if (unlikely(ret))
|
| 5830 | goto out;
|
| 5831 | }
|
| 5832 |
|
| 5833 | folio = alloc_hugetlb_folio(vma, vmf->address, false);
|
| 5834 | if (IS_ERR(folio)) {
|
| 5835 | |
| 5836 | * Returning error will result in faulting task being
|
| 5837 | * sent SIGBUS. The hugetlb fault mutex prevents two
|
| 5838 | * tasks from racing to fault in the same page which
|
| 5839 | * could result in false unable to allocate errors.
|
| 5840 | * Page migration does not take the fault mutex, but
|
| 5841 | * does a clear then write of pte's under page table
|
| 5842 | * lock. Page fault code could race with migration,
|
| 5843 | * notice the clear pte and try to allocate a page
|
| 5844 | * here. Before returning error, get ptl and make
|
| 5845 | * sure there really is no pte entry.
|
| 5846 |
|
| 5847 | if (hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte))
|
| 5848 | ret = vmf_error(PTR_ERR(folio));
|
| 5849 | else
|
| 5850 | ret = 0;
|
| 5851 | goto out;
|
| 5852 | }
|
| 5853 | folio_zero_user(folio, vmf->real_address);
|
| 5854 | __folio_mark_uptodate(folio);
|
| 5855 | new_folio = true;
|
| 5856 |
|
| 5857 | if (vma->vm_flags & VM_MAYSHARE) {
|
| 5858 | int err = hugetlb_add_to_page_cache(folio, mapping,
|
| 5859 | vmf->pgoff);
|
| 5860 | if (err) {
|
| 5861 | |
| 5862 | * err can't be -EEXIST which implies someone
|
| 5863 | * else consumed the reservation since hugetlb
|
| 5864 | * fault mutex is held when add a hugetlb page
|
| 5865 | * to the page cache. So it's safe to call
|
| 5866 | * restore_reserve_on_error() here.
|
| 5867 |
|
| 5868 | restore_reserve_on_error(h, vma, vmf->address,
|
| 5869 | folio);
|
| 5870 | folio_put(folio);
|
| 5871 | ret = VM_FAULT_SIGBUS;
|
| 5872 | goto out;
|
| 5873 | }
|
| 5874 | } else {
|
| 5875 | new_anon_folio = true;
|
| 5876 | folio_lock(folio);
|
| 5877 | }
|
| 5878 | } else {
|
| 5879 | |
| 5880 | * If memory error occurs between mmap() and fault, some process
|
| 5881 | * don't have hwpoisoned swap entry for errored virtual address.
|
| 5882 | * So we need to block hugepage fault by PG_hwpoison bit check.
|
| 5883 |
|
| 5884 | if (unlikely(folio_test_hwpoison(folio))) {
|
| 5885 | ret = VM_FAULT_HWPOISON_LARGE |
|
| 5886 | VM_FAULT_SET_HINDEX(hstate_index(h));
|
| 5887 | goto backout_unlocked;
|
| 5888 | }
|
| 5889 |
|
| 5890 |
|
| 5891 | if (userfaultfd_minor(vma)) {
|
| 5892 | folio_unlock(folio);
|
| 5893 | folio_put(folio);
|
| 5894 |
|
| 5895 | if (!hugetlb_pte_stable(h, mm, vmf->address, vmf->pte, vmf->orig_pte)) {
|
| 5896 | ret = 0;
|
| 5897 | goto out;
|
| 5898 | }
|
| 5899 | return hugetlb_handle_userfault(vmf, mapping,
|
| 5900 | VM_UFFD_MINOR);
|
| 5901 | }
|
| 5902 | }
|
| 5903 |
|
| 5904 | |
| 5905 | * If we are going to COW a private mapping later, we examine the
|
| 5906 | * pending reservations for this page now. This will ensure that
|
| 5907 | * any allocations necessary to record that reservation occur outside
|
| 5908 | * the spinlock.
|
| 5909 |
|
| 5910 | if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
|
| 5911 | if (vma_needs_reservation(h, vma, vmf->address) < 0) {
|
| 5912 | ret = VM_FAULT_OOM;
|
| 5913 | goto backout_unlocked;
|
| 5914 | }
|
| 5915 |
|
| 5916 | vma_end_reservation(h, vma, vmf->address);
|
| 5917 | }
|
| 5918 |
|
| 5919 | vmf->ptl = huge_pte_lock(h, mm, vmf->pte);
|
| 5920 | ret = 0;
|
| 5921 |
|
| 5922 | if (!pte_same(huge_ptep_get(mm, vmf->address, vmf->pte), vmf->orig_pte))
|
| 5923 | goto backout;
|
| 5924 |
|
| 5925 | if (new_anon_folio)
|
| 5926 | hugetlb_add_new_anon_rmap(folio, vma, vmf->address);
|
| 5927 | else
|
| 5928 | hugetlb_add_file_rmap(folio);
|
| 5929 | new_pte = make_huge_pte(vma, folio, vma->vm_flags & VM_SHARED);
|
| 5930 | |
| 5931 | * If this pte was previously wr-protected, keep it wr-protected even
|
| 5932 | * if populated.
|
| 5933 |
|
| 5934 | if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte)))
|
| 5935 | new_pte = huge_pte_mkuffd(new_pte);
|
| 5936 | set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h));
|
| 5937 |
|
| 5938 | hugetlb_count_add(pages_per_huge_page(h), mm);
|
| 5939 | if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
|
| 5940 | |
| 5941 | * No need to keep file folios locked. See comment in
|
| 5942 | * hugetlb_fault().
|
| 5943 |
|
| 5944 | if (!new_anon_folio) {
|
| 5945 | folio_locked = false;
|
| 5946 | folio_unlock(folio);
|
| 5947 | }
|
| 5948 |
|
| 5949 | ret = hugetlb_wp(vmf);
|
| 5950 | }
|
| 5951 |
|
| 5952 | spin_unlock(vmf->ptl);
|
| 5953 |
|
| 5954 | |
| 5955 | * Only set hugetlb_migratable in newly allocated pages. Existing pages
|
| 5956 | * found in the pagecache may not have hugetlb_migratable if they have
|
| 5957 | * been isolated for migration.
|
| 5958 |
|
| 5959 | if (new_folio)
|
| 5960 | folio_set_hugetlb_migratable(folio);
|
| 5961 |
|
| 5962 | if (folio_locked)
|
| 5963 | folio_unlock(folio);
|
| 5964 | out:
|
| 5965 | hugetlb_vma_unlock_read(vma);
|
| 5966 |
|
| 5967 | |
| 5968 | * We must check to release the per-VMA lock. __vmf_anon_prepare() is
|
| 5969 | * the only way ret can be set to VM_FAULT_RETRY.
|
| 5970 |
|
| 5971 | if (unlikely(ret & VM_FAULT_RETRY))
|
| 5972 | vma_end_read(vma);
|
| 5973 |
|
| 5974 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 5975 | return ret;
|
| 5976 |
|
| 5977 | backout:
|
| 5978 | spin_unlock(vmf->ptl);
|
| 5979 | backout_unlocked:
|
| 5980 |
|
| 5981 | if (new_anon_folio)
|
| 5982 | restore_reserve_on_error(h, vma, vmf->address, folio);
|
| 5983 |
|
| 5984 | folio_unlock(folio);
|
| 5985 | folio_put(folio);
|
| 5986 | goto out;
|
| 5987 | }
|
| 5988 |
|
| 5989 | #ifdef CONFIG_SMP
|
| 5990 | u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
|
| 5991 | {
|
| 5992 | unsigned long key[2];
|
| 5993 | u32 hash;
|
| 5994 |
|
| 5995 | key[0] = (unsigned long) mapping;
|
| 5996 | key[1] = idx;
|
| 5997 |
|
| 5998 | hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
|
| 5999 |
|
| 6000 | return hash & (num_fault_mutexes - 1);
|
| 6001 | }
|
| 6002 | #else
|
| 6003 | |
| 6004 | * For uniprocessor systems we always use a single mutex, so just
|
| 6005 | * return 0 and avoid the hashing overhead.
|
| 6006 |
|
| 6007 | u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
|
| 6008 | {
|
| 6009 | return 0;
|
| 6010 | }
|
| 6011 | #endif
|
| 6012 |
|
| 6013 | vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
|
| 6014 | unsigned long address, unsigned int flags)
|
| 6015 | {
|
| 6016 | vm_fault_t ret;
|
| 6017 | u32 hash;
|
| 6018 | struct folio *folio = NULL;
|
| 6019 | struct hstate *h = hstate_vma(vma);
|
| 6020 | struct address_space *mapping;
|
| 6021 | bool need_wait_lock = false;
|
| 6022 | struct vm_fault vmf = {
|
| 6023 | .vma = vma,
|
| 6024 | .address = address & huge_page_mask(h),
|
| 6025 | .real_address = address,
|
| 6026 | .flags = flags,
|
| 6027 | .pgoff = vma_hugecache_offset(h, vma,
|
| 6028 | address & huge_page_mask(h)),
|
| 6029 |
|
| 6030 |
|
| 6031 | |
| 6032 | * Some fields may not be initialized, be careful as it may
|
| 6033 | * be hard to debug if called functions make assumptions
|
| 6034 |
|
| 6035 | };
|
| 6036 |
|
| 6037 | |
| 6038 | * Serialize hugepage allocation and instantiation, so that we don't
|
| 6039 | * get spurious allocation failures if two CPUs race to instantiate
|
| 6040 | * the same page in the page cache.
|
| 6041 |
|
| 6042 | mapping = vma->vm_file->f_mapping;
|
| 6043 | hash = hugetlb_fault_mutex_hash(mapping, vmf.pgoff);
|
| 6044 | mutex_lock(&hugetlb_fault_mutex_table[hash]);
|
| 6045 |
|
| 6046 | |
| 6047 | * Acquire vma lock before calling huge_pte_alloc and hold
|
| 6048 | * until finished with vmf.pte. This prevents huge_pmd_unshare from
|
| 6049 | * being called elsewhere and making the vmf.pte no longer valid.
|
| 6050 |
|
| 6051 | hugetlb_vma_lock_read(vma);
|
| 6052 | vmf.pte = huge_pte_alloc(mm, vma, vmf.address, huge_page_size(h));
|
| 6053 | if (!vmf.pte) {
|
| 6054 | hugetlb_vma_unlock_read(vma);
|
| 6055 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 6056 | return VM_FAULT_OOM;
|
| 6057 | }
|
| 6058 |
|
| 6059 | vmf.orig_pte = huge_ptep_get(mm, vmf.address, vmf.pte);
|
| 6060 | if (huge_pte_none(vmf.orig_pte))
|
| 6061 | |
| 6062 | * hugetlb_no_page will drop vma lock and hugetlb fault
|
| 6063 | * mutex internally, which make us return immediately.
|
| 6064 |
|
| 6065 | return hugetlb_no_page(mapping, &vmf);
|
| 6066 |
|
| 6067 | if (pte_is_marker(vmf.orig_pte)) {
|
| 6068 | const pte_marker marker =
|
| 6069 | softleaf_to_marker(softleaf_from_pte(vmf.orig_pte));
|
| 6070 |
|
| 6071 | if (marker & PTE_MARKER_POISONED) {
|
| 6072 | ret = VM_FAULT_HWPOISON_LARGE |
|
| 6073 | VM_FAULT_SET_HINDEX(hstate_index(h));
|
| 6074 | goto out_mutex;
|
| 6075 | } else if (WARN_ON_ONCE(marker & PTE_MARKER_GUARD)) {
|
| 6076 |
|
| 6077 | ret = VM_FAULT_SIGSEGV;
|
| 6078 | goto out_mutex;
|
| 6079 | }
|
| 6080 |
|
| 6081 | return hugetlb_no_page(mapping, &vmf);
|
| 6082 | }
|
| 6083 |
|
| 6084 | ret = 0;
|
| 6085 |
|
| 6086 |
|
| 6087 | if (!pte_present(vmf.orig_pte) && !huge_pte_none(vmf.orig_pte)) {
|
| 6088 | const softleaf_t softleaf = softleaf_from_pte(vmf.orig_pte);
|
| 6089 |
|
| 6090 | if (softleaf_is_migration(softleaf)) {
|
| 6091 | |
| 6092 | * Release the hugetlb fault lock now, but retain
|
| 6093 | * the vma lock, because it is needed to guard the
|
| 6094 | * huge_pte_lockptr() later in
|
| 6095 | * migration_entry_wait_huge(). The vma lock will
|
| 6096 | * be released there.
|
| 6097 |
|
| 6098 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 6099 | migration_entry_wait_huge(vma, vmf.address, vmf.pte);
|
| 6100 | return 0;
|
| 6101 | }
|
| 6102 | if (softleaf_is_hwpoison(softleaf)) {
|
| 6103 | ret = VM_FAULT_HWPOISON_LARGE |
|
| 6104 | VM_FAULT_SET_HINDEX(hstate_index(h));
|
| 6105 | }
|
| 6106 |
|
| 6107 | goto out_mutex;
|
| 6108 | }
|
| 6109 |
|
| 6110 | |
| 6111 | * Protnone hugetlb PTEs with the uffd bit are used by
|
| 6112 | * userfaultfd RWP for access tracking. Plain PROT_NONE (without the
|
| 6113 | * marker) is not an RWP fault and is not expected on hugetlb (no
|
| 6114 | * NUMA hinting), so let normal hugetlb fault handling proceed.
|
| 6115 |
|
| 6116 | if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) &&
|
| 6117 | userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) {
|
| 6118 | spinlock_t *ptl;
|
| 6119 | pte_t pte;
|
| 6120 |
|
| 6121 |
|
| 6122 | if (!userfaultfd_rwp_async(vma))
|
| 6123 | return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP);
|
| 6124 |
|
| 6125 | ptl = huge_pte_lock(h, mm, vmf.pte);
|
| 6126 | pte = huge_ptep_get(mm, vmf.address, vmf.pte);
|
| 6127 | if (pte_protnone(pte) && huge_pte_uffd(pte)) {
|
| 6128 | unsigned int shift = huge_page_shift(h);
|
| 6129 |
|
| 6130 | pte = huge_pte_modify(pte, vma->vm_page_prot);
|
| 6131 | pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
|
| 6132 |
|
| 6133 | pte = huge_pte_clear_uffd(pte);
|
| 6134 | pte = pte_mkyoung(pte);
|
| 6135 | |
| 6136 | * Unlike do_uffd_rwp(), do not upgrade to writable
|
| 6137 | * here. Hugetlb lacks a can_change_huge_pte_writable()
|
| 6138 | * equivalent, so a write access will take a separate
|
| 6139 | * COW fault — acceptable for the rare private hugetlb
|
| 6140 | * case.
|
| 6141 |
|
| 6142 | set_huge_pte_at(mm, vmf.address, vmf.pte, pte,
|
| 6143 | huge_page_size(h));
|
| 6144 | update_mmu_cache(vma, vmf.address, vmf.pte);
|
| 6145 | }
|
| 6146 | spin_unlock(ptl);
|
| 6147 | ret = 0;
|
| 6148 | goto out_mutex;
|
| 6149 | }
|
| 6150 |
|
| 6151 | |
| 6152 | * If we are going to COW/unshare the mapping later, we examine the
|
| 6153 | * pending reservations for this page now. This will ensure that any
|
| 6154 | * allocations necessary to record that reservation occur outside the
|
| 6155 | * spinlock.
|
| 6156 |
|
| 6157 | if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
|
| 6158 | !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf.orig_pte)) {
|
| 6159 | if (vma_needs_reservation(h, vma, vmf.address) < 0) {
|
| 6160 | ret = VM_FAULT_OOM;
|
| 6161 | goto out_mutex;
|
| 6162 | }
|
| 6163 |
|
| 6164 | vma_end_reservation(h, vma, vmf.address);
|
| 6165 | }
|
| 6166 |
|
| 6167 | vmf.ptl = huge_pte_lock(h, mm, vmf.pte);
|
| 6168 |
|
| 6169 |
|
| 6170 | if (unlikely(!pte_same(vmf.orig_pte, huge_ptep_get(mm, vmf.address, vmf.pte))))
|
| 6171 | goto out_ptl;
|
| 6172 |
|
| 6173 |
|
| 6174 | if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) &&
|
| 6175 | (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) {
|
| 6176 | if (!userfaultfd_wp_async(vma)) {
|
| 6177 | spin_unlock(vmf.ptl);
|
| 6178 | hugetlb_vma_unlock_read(vma);
|
| 6179 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 6180 | return handle_userfault(&vmf, VM_UFFD_WP);
|
| 6181 | }
|
| 6182 |
|
| 6183 | vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte);
|
| 6184 | set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte,
|
| 6185 | huge_page_size(hstate_vma(vma)));
|
| 6186 |
|
| 6187 | }
|
| 6188 |
|
| 6189 | if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
|
| 6190 | if (!huge_pte_write(vmf.orig_pte)) {
|
| 6191 | |
| 6192 | * Anonymous folios need to be lock since hugetlb_wp()
|
| 6193 | * checks whether we can re-use the folio exclusively
|
| 6194 | * for us in case we are the only user of it.
|
| 6195 |
|
| 6196 | folio = page_folio(pte_page(vmf.orig_pte));
|
| 6197 | if (folio_test_anon(folio) && !folio_trylock(folio)) {
|
| 6198 | need_wait_lock = true;
|
| 6199 | goto out_ptl;
|
| 6200 | }
|
| 6201 | folio_get(folio);
|
| 6202 | ret = hugetlb_wp(&vmf);
|
| 6203 | if (folio_test_anon(folio))
|
| 6204 | folio_unlock(folio);
|
| 6205 | folio_put(folio);
|
| 6206 | goto out_ptl;
|
| 6207 | } else if (likely(flags & FAULT_FLAG_WRITE)) {
|
| 6208 | vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
|
| 6209 | }
|
| 6210 | }
|
| 6211 | vmf.orig_pte = pte_mkyoung(vmf.orig_pte);
|
| 6212 | if (huge_ptep_set_access_flags(vma, vmf.address, vmf.pte, vmf.orig_pte,
|
| 6213 | flags & FAULT_FLAG_WRITE))
|
| 6214 | update_mmu_cache(vma, vmf.address, vmf.pte);
|
| 6215 | out_ptl:
|
| 6216 | spin_unlock(vmf.ptl);
|
| 6217 | out_mutex:
|
| 6218 | hugetlb_vma_unlock_read(vma);
|
| 6219 |
|
| 6220 | |
| 6221 | * We must check to release the per-VMA lock. __vmf_anon_prepare() in
|
| 6222 | * hugetlb_wp() is the only way ret can be set to VM_FAULT_RETRY.
|
| 6223 |
|
| 6224 | if (unlikely(ret & VM_FAULT_RETRY))
|
| 6225 | vma_end_read(vma);
|
| 6226 |
|
| 6227 | mutex_unlock(&hugetlb_fault_mutex_table[hash]);
|
| 6228 | |
| 6229 | * hugetlb_wp drops all the locks, but the folio lock, before trying to
|
| 6230 | * unmap the folio from other processes. During that window, if another
|
| 6231 | * process mapping that folio faults in, it will take the mutex and then
|
| 6232 | * it will wait on folio_lock, causing an ABBA deadlock.
|
| 6233 | * Use trylock instead and bail out if we fail.
|
| 6234 | *
|
| 6235 | * Ideally, we should hold a refcount on the folio we wait for, but we do
|
| 6236 | * not want to use the folio after it becomes unlocked, but rather just
|
| 6237 | * wait for it to become unlocked, so hopefully next fault successes on
|
| 6238 | * the trylock.
|
| 6239 |
|
| 6240 | if (need_wait_lock)
|
| 6241 | folio_wait_locked(folio);
|
| 6242 | return ret;
|
| 6243 | }
|
| 6244 |
|
| 6245 | #ifdef CONFIG_USERFAULTFD
|
| 6246 | |
| 6247 | * Can probably be eliminated, but still used by hugetlb_mfill_atomic_pte().
|
| 6248 |
|
| 6249 | static struct folio *alloc_hugetlb_folio_vma(struct hstate *h,
|
| 6250 | struct vm_area_struct *vma, unsigned long address)
|
| 6251 | {
|
| 6252 | struct mempolicy *mpol;
|
| 6253 | nodemask_t *nodemask;
|
| 6254 | struct folio *folio;
|
| 6255 | gfp_t gfp_mask;
|
| 6256 | int node;
|
| 6257 |
|
| 6258 | gfp_mask = htlb_alloc_mask(h);
|
| 6259 | node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
|
| 6260 | |
| 6261 | * This is used to allocate a temporary hugetlb to hold the copied
|
| 6262 | * content, which will then be copied again to the final hugetlb
|
| 6263 | * consuming a reservation. Set the alloc_fallback to false to indicate
|
| 6264 | * that breaking the per-node hugetlb pool is not allowed in this case.
|
| 6265 |
|
| 6266 | folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask, false);
|
| 6267 | mpol_cond_put(mpol);
|
| 6268 |
|
| 6269 | return folio;
|
| 6270 | }
|
| 6271 |
|
| 6272 | |
| 6273 | * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte
|
| 6274 | * with modifications for hugetlb pages.
|
| 6275 |
|
| 6276 | int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
|
| 6277 | struct vm_area_struct *dst_vma,
|
| 6278 | unsigned long dst_addr,
|
| 6279 | unsigned long src_addr,
|
| 6280 | uffd_flags_t flags,
|
| 6281 | struct folio **foliop)
|
| 6282 | {
|
| 6283 | struct mm_struct *dst_mm = dst_vma->vm_mm;
|
| 6284 | bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE);
|
| 6285 | bool wp_enabled = (flags & MFILL_ATOMIC_WP);
|
| 6286 | struct hstate *h = hstate_vma(dst_vma);
|
| 6287 | struct address_space *mapping = dst_vma->vm_file->f_mapping;
|
| 6288 | pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
|
| 6289 | unsigned long size = huge_page_size(h);
|
| 6290 | int vm_shared = dst_vma->vm_flags & VM_SHARED;
|
| 6291 | pte_t _dst_pte;
|
| 6292 | spinlock_t *ptl;
|
| 6293 | int ret = -ENOMEM;
|
| 6294 | struct folio *folio;
|
| 6295 | bool folio_in_pagecache = false;
|
| 6296 | pte_t dst_ptep;
|
| 6297 |
|
| 6298 | if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) {
|
| 6299 | ptl = huge_pte_lock(h, dst_mm, dst_pte);
|
| 6300 |
|
| 6301 |
|
| 6302 | if (!huge_pte_none(huge_ptep_get(dst_mm, dst_addr, dst_pte))) {
|
| 6303 | spin_unlock(ptl);
|
| 6304 | return -EEXIST;
|
| 6305 | }
|
| 6306 |
|
| 6307 | _dst_pte = make_pte_marker(PTE_MARKER_POISONED);
|
| 6308 | set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);
|
| 6309 |
|
| 6310 |
|
| 6311 | update_mmu_cache(dst_vma, dst_addr, dst_pte);
|
| 6312 |
|
| 6313 | spin_unlock(ptl);
|
| 6314 | return 0;
|
| 6315 | }
|
| 6316 |
|
| 6317 | if (is_continue) {
|
| 6318 | ret = -EFAULT;
|
| 6319 | folio = filemap_lock_hugetlb_folio(h, mapping, idx);
|
| 6320 | if (IS_ERR(folio))
|
| 6321 | goto out;
|
| 6322 | folio_in_pagecache = true;
|
| 6323 | } else if (!*foliop) {
|
| 6324 | |
| 6325 | * a non-missing case. Return -EEXIST.
|
| 6326 |
|
| 6327 | if (vm_shared &&
|
| 6328 | hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
|
| 6329 | ret = -EEXIST;
|
| 6330 | goto out;
|
| 6331 | }
|
| 6332 |
|
| 6333 | folio = alloc_hugetlb_folio(dst_vma, dst_addr, false);
|
| 6334 | if (IS_ERR(folio)) {
|
| 6335 | pte_t *actual_pte = hugetlb_walk(dst_vma, dst_addr, PMD_SIZE);
|
| 6336 | if (actual_pte) {
|
| 6337 | ret = -EEXIST;
|
| 6338 | goto out;
|
| 6339 | }
|
| 6340 | ret = -ENOMEM;
|
| 6341 | goto out;
|
| 6342 | }
|
| 6343 |
|
| 6344 | ret = copy_folio_from_user(folio, (const void __user *) src_addr,
|
| 6345 | false);
|
| 6346 |
|
| 6347 |
|
| 6348 | if (unlikely(ret)) {
|
| 6349 | ret = -ENOENT;
|
| 6350 | |
| 6351 | * consumed a reservation.
|
| 6352 |
|
| 6353 | restore_reserve_on_error(h, dst_vma, dst_addr, folio);
|
| 6354 | folio_put(folio);
|
| 6355 |
|
| 6356 | |
| 6357 | * contents.
|
| 6358 |
|
| 6359 | folio = alloc_hugetlb_folio_vma(h, dst_vma, dst_addr);
|
| 6360 | if (!folio) {
|
| 6361 | ret = -ENOMEM;
|
| 6362 | goto out;
|
| 6363 | }
|
| 6364 | *foliop = folio;
|
| 6365 | |
| 6366 | * copy the contents outside the lock. Don't free the
|
| 6367 | * folio.
|
| 6368 |
|
| 6369 | goto out;
|
| 6370 | }
|
| 6371 | } else {
|
| 6372 | if (vm_shared &&
|
| 6373 | hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
|
| 6374 | folio_put(*foliop);
|
| 6375 | ret = -EEXIST;
|
| 6376 | *foliop = NULL;
|
| 6377 | goto out;
|
| 6378 | }
|
| 6379 |
|
| 6380 | folio = alloc_hugetlb_folio(dst_vma, dst_addr, false);
|
| 6381 | if (IS_ERR(folio)) {
|
| 6382 | folio_put(*foliop);
|
| 6383 | ret = -ENOMEM;
|
| 6384 | *foliop = NULL;
|
| 6385 | goto out;
|
| 6386 | }
|
| 6387 | ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma);
|
| 6388 | folio_put(*foliop);
|
| 6389 | *foliop = NULL;
|
| 6390 | if (ret) {
|
| 6391 | restore_reserve_on_error(h, dst_vma, dst_addr, folio);
|
| 6392 | folio_put(folio);
|
| 6393 | goto out;
|
| 6394 | }
|
| 6395 | }
|
| 6396 |
|
| 6397 | |
| 6398 | * If we just allocated a new page, we need a memory barrier to ensure
|
| 6399 | * that preceding stores to the page become visible before the
|
| 6400 | * set_pte_at() write. The memory barrier inside __folio_mark_uptodate
|
| 6401 | * is what we need.
|
| 6402 | *
|
| 6403 | * In the case where we have not allocated a new page (is_continue),
|
| 6404 | * the page must already be uptodate. UFFDIO_CONTINUE already includes
|
| 6405 | * an earlier smp_wmb() to ensure that prior stores will be visible
|
| 6406 | * before the set_pte_at() write.
|
| 6407 |
|
| 6408 | if (!is_continue)
|
| 6409 | __folio_mark_uptodate(folio);
|
| 6410 | else
|
| 6411 | WARN_ON_ONCE(!folio_test_uptodate(folio));
|
| 6412 |
|
| 6413 |
|
| 6414 | if (vm_shared && !is_continue) {
|
| 6415 | ret = -EFAULT;
|
| 6416 | if (idx >= (i_size_read(mapping->host) >> huge_page_shift(h)))
|
| 6417 | goto out_release_nounlock;
|
| 6418 |
|
| 6419 | |
| 6420 | * Serialization between remove_inode_hugepages() and
|
| 6421 | * hugetlb_add_to_page_cache() below happens through the
|
| 6422 | * hugetlb_fault_mutex_table that here must be hold by
|
| 6423 | * the caller.
|
| 6424 |
|
| 6425 | ret = hugetlb_add_to_page_cache(folio, mapping, idx);
|
| 6426 | if (ret)
|
| 6427 | goto out_release_nounlock;
|
| 6428 | folio_in_pagecache = true;
|
| 6429 | }
|
| 6430 |
|
| 6431 | ptl = huge_pte_lock(h, dst_mm, dst_pte);
|
| 6432 |
|
| 6433 | ret = -EIO;
|
| 6434 | if (folio_test_hwpoison(folio))
|
| 6435 | goto out_release_unlock;
|
| 6436 |
|
| 6437 | ret = -EEXIST;
|
| 6438 |
|
| 6439 | dst_ptep = huge_ptep_get(dst_mm, dst_addr, dst_pte);
|
| 6440 | |
| 6441 | * See comment about UFFD marker overwriting in
|
| 6442 | * mfill_atomic_install_pte().
|
| 6443 |
|
| 6444 | if (!huge_pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep))
|
| 6445 | goto out_release_unlock;
|
| 6446 |
|
| 6447 | if (folio_in_pagecache)
|
| 6448 | hugetlb_add_file_rmap(folio);
|
| 6449 | else
|
| 6450 | hugetlb_add_new_anon_rmap(folio, dst_vma, dst_addr);
|
| 6451 |
|
| 6452 | |
| 6453 | * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
|
| 6454 | * with wp flag set, don't set pte write bit.
|
| 6455 |
|
| 6456 | _dst_pte = make_huge_pte(dst_vma, folio,
|
| 6457 | !wp_enabled && !(is_continue && !vm_shared));
|
| 6458 | |
| 6459 | * Always mark UFFDIO_COPY page dirty; note that this may not be
|
| 6460 | * extremely important for hugetlbfs for now since swapping is not
|
| 6461 | * supported, but we should still be clear in that this page cannot be
|
| 6462 | * thrown away at will, even if write bit not set.
|
| 6463 |
|
| 6464 | _dst_pte = huge_pte_mkdirty(_dst_pte);
|
| 6465 | _dst_pte = pte_mkyoung(_dst_pte);
|
| 6466 |
|
| 6467 | if (wp_enabled)
|
| 6468 | _dst_pte = huge_pte_mkuffd(_dst_pte);
|
| 6469 |
|
| 6470 | set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);
|
| 6471 |
|
| 6472 | hugetlb_count_add(pages_per_huge_page(h), dst_mm);
|
| 6473 |
|
| 6474 |
|
| 6475 | update_mmu_cache(dst_vma, dst_addr, dst_pte);
|
| 6476 |
|
| 6477 | spin_unlock(ptl);
|
| 6478 | if (!is_continue)
|
| 6479 | folio_set_hugetlb_migratable(folio);
|
| 6480 | if (vm_shared || is_continue)
|
| 6481 | folio_unlock(folio);
|
| 6482 | ret = 0;
|
| 6483 | out:
|
| 6484 | return ret;
|
| 6485 | out_release_unlock:
|
| 6486 | spin_unlock(ptl);
|
| 6487 | if (vm_shared || is_continue)
|
| 6488 | folio_unlock(folio);
|
| 6489 | out_release_nounlock:
|
| 6490 | if (!folio_in_pagecache)
|
| 6491 | restore_reserve_on_error(h, dst_vma, dst_addr, folio);
|
| 6492 | folio_put(folio);
|
| 6493 | goto out;
|
| 6494 | }
|
| 6495 | #endif
|
| 6496 |
|
| 6497 | long hugetlb_change_protection(struct vm_area_struct *vma,
|
| 6498 | unsigned long address, unsigned long end,
|
| 6499 | pgprot_t newprot, unsigned long cp_flags)
|
| 6500 | {
|
| 6501 | struct mm_struct *mm = vma->vm_mm;
|
| 6502 | unsigned long start = address;
|
| 6503 | pte_t *ptep;
|
| 6504 | pte_t pte;
|
| 6505 | struct hstate *h = hstate_vma(vma);
|
| 6506 | long pages = 0, psize = huge_page_size(h);
|
| 6507 | struct mmu_notifier_range range;
|
| 6508 | unsigned long last_addr_mask;
|
| 6509 | bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
|
| 6510 | bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
|
| 6511 | bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
|
| 6512 | bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE;
|
| 6513 | struct mmu_gather tlb;
|
| 6514 |
|
| 6515 | |
| 6516 | * In the case of shared PMDs, the area to flush could be beyond
|
| 6517 | * start/end. Set range.start/range.end to cover the maximum possible
|
| 6518 | * range if PMD sharing is possible.
|
| 6519 |
|
| 6520 | mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
|
| 6521 | 0, mm, start, end);
|
| 6522 | adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
|
| 6523 |
|
| 6524 | BUG_ON(address >= end);
|
| 6525 | flush_cache_range(vma, range.start, range.end);
|
| 6526 | tlb_gather_mmu_vma(&tlb, vma);
|
| 6527 |
|
| 6528 | mmu_notifier_invalidate_range_start(&range);
|
| 6529 | hugetlb_vma_lock_write(vma);
|
| 6530 | i_mmap_lock_write(vma->vm_file->f_mapping);
|
| 6531 | last_addr_mask = hugetlb_mask_last_page(h);
|
| 6532 | for (; address < end; address += psize) {
|
| 6533 | softleaf_t entry;
|
| 6534 | spinlock_t *ptl;
|
| 6535 |
|
| 6536 | ptep = hugetlb_walk(vma, address, psize);
|
| 6537 | if (!ptep) {
|
| 6538 | |
| 6539 | * uffd_wp installs a pte marker on the unpopulated
|
| 6540 | * entry; uffd_rwp does not install markers so the
|
| 6541 | * allocation is unnecessary for it.
|
| 6542 |
|
| 6543 | if (!uffd_wp) {
|
| 6544 | address |= last_addr_mask;
|
| 6545 | continue;
|
| 6546 | }
|
| 6547 | |
| 6548 | * Userfaultfd wr-protect requires pgtable
|
| 6549 | * pre-allocations to install pte markers.
|
| 6550 |
|
| 6551 | ptep = huge_pte_alloc(mm, vma, address, psize);
|
| 6552 | if (!ptep) {
|
| 6553 | pages = -ENOMEM;
|
| 6554 | break;
|
| 6555 | }
|
| 6556 | }
|
| 6557 | ptl = huge_pte_lock(h, mm, ptep);
|
| 6558 | if (huge_pmd_unshare(&tlb, vma, address, ptep)) {
|
| 6559 | |
| 6560 | * When uffd-wp is enabled on the vma, unshare
|
| 6561 | * shouldn't happen at all. Warn about it if it
|
| 6562 | * happened due to some reason.
|
| 6563 |
|
| 6564 | WARN_ON_ONCE(uffd_wp || uffd_wp_resolve ||
|
| 6565 | uffd_rwp || uffd_rwp_resolve);
|
| 6566 | pages++;
|
| 6567 | spin_unlock(ptl);
|
| 6568 | address |= last_addr_mask;
|
| 6569 | continue;
|
| 6570 | }
|
| 6571 | pte = huge_ptep_get(mm, address, ptep);
|
| 6572 | if (huge_pte_none(pte)) {
|
| 6573 | if (unlikely(uffd_wp))
|
| 6574 |
|
| 6575 | set_huge_pte_at(mm, address, ptep,
|
| 6576 | make_pte_marker(PTE_MARKER_UFFD_WP),
|
| 6577 | psize);
|
| 6578 | goto next;
|
| 6579 | }
|
| 6580 |
|
| 6581 | entry = softleaf_from_pte(pte);
|
| 6582 | if (unlikely(softleaf_is_hwpoison(entry))) {
|
| 6583 |
|
| 6584 | } else if (unlikely(softleaf_is_migration(entry))) {
|
| 6585 | struct folio *folio = softleaf_to_folio(entry);
|
| 6586 | pte_t newpte = pte;
|
| 6587 |
|
| 6588 | if (softleaf_is_migration_write(entry)) {
|
| 6589 | if (folio_test_anon(folio))
|
| 6590 | entry = make_readable_exclusive_migration_entry(
|
| 6591 | swp_offset(entry));
|
| 6592 | else
|
| 6593 | entry = make_readable_migration_entry(
|
| 6594 | swp_offset(entry));
|
| 6595 | newpte = swp_entry_to_pte(entry);
|
| 6596 | pages++;
|
| 6597 | }
|
| 6598 |
|
| 6599 | if (uffd_wp || uffd_rwp)
|
| 6600 | newpte = pte_swp_mkuffd(newpte);
|
| 6601 | else if (uffd_wp_resolve || uffd_rwp_resolve)
|
| 6602 | newpte = pte_swp_clear_uffd(newpte);
|
| 6603 | if (!pte_same(pte, newpte))
|
| 6604 | set_huge_pte_at(mm, address, ptep, newpte, psize);
|
| 6605 | } else if (unlikely(pte_is_marker(pte))) {
|
| 6606 | |
| 6607 | * Do nothing on a poison marker; page is
|
| 6608 | * corrupted, permissions do not apply. Here
|
| 6609 | * pte_marker_uffd_wp()==true implies !poison
|
| 6610 | * because they're mutual exclusive.
|
| 6611 |
|
| 6612 | if (pte_is_uffd_wp_marker(pte) &&
|
| 6613 | (uffd_wp_resolve || uffd_rwp_resolve))
|
| 6614 |
|
| 6615 | huge_pte_clear(mm, address, ptep, psize);
|
| 6616 | } else {
|
| 6617 | pte_t old_pte;
|
| 6618 | unsigned int shift = huge_page_shift(hstate_vma(vma));
|
| 6619 |
|
| 6620 |
|
| 6621 | if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte))
|
| 6622 | goto next;
|
| 6623 |
|
| 6624 | old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
|
| 6625 | pte = huge_pte_modify(old_pte, newprot);
|
| 6626 | pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
|
| 6627 | if (uffd_wp || uffd_rwp)
|
| 6628 | pte = huge_pte_mkuffd(pte);
|
| 6629 | else if (uffd_wp_resolve || uffd_rwp_resolve)
|
| 6630 | pte = huge_pte_clear_uffd(pte);
|
| 6631 |
|
| 6632 |
|
| 6633 | if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
|
| 6634 | pte = huge_pte_modify(pte, PAGE_NONE);
|
| 6635 | pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
|
| 6636 | }
|
| 6637 |
|
| 6638 | huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
|
| 6639 | pages++;
|
| 6640 | tlb_remove_huge_tlb_entry(h, &tlb, ptep, address);
|
| 6641 | }
|
| 6642 |
|
| 6643 | next:
|
| 6644 | spin_unlock(ptl);
|
| 6645 | cond_resched();
|
| 6646 | }
|
| 6647 |
|
| 6648 | tlb_flush_mmu_tlbonly(&tlb);
|
| 6649 | huge_pmd_unshare_flush(&tlb, vma);
|
| 6650 | |
| 6651 | * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are
|
| 6652 | * downgrading page table protection not changing it to point to a new
|
| 6653 | * page.
|
| 6654 | *
|
| 6655 | * See Documentation/mm/mmu_notifier.rst
|
| 6656 |
|
| 6657 | i_mmap_unlock_write(vma->vm_file->f_mapping);
|
| 6658 | hugetlb_vma_unlock_write(vma);
|
| 6659 | mmu_notifier_invalidate_range_end(&range);
|
| 6660 | tlb_finish_mmu(&tlb);
|
| 6661 |
|
| 6662 | return pages > 0 ? (pages << h->order) : pages;
|
| 6663 | }
|
| 6664 |
|
| 6665 | |
| 6666 | * Update the reservation map for the range [from, to].
|
| 6667 | *
|
| 6668 | * Returns the number of entries that would be added to the reservation map
|
| 6669 | * associated with the range [from, to]. This number is greater or equal to
|
| 6670 | * zero. -EINVAL or -ENOMEM is returned in case of any errors.
|
| 6671 |
|
| 6672 |
|
| 6673 | long hugetlb_reserve_pages(struct inode *inode,
|
| 6674 | long from, long to,
|
| 6675 | struct vm_area_struct *vma,
|
| 6676 | vma_flags_t vma_flags)
|
| 6677 | {
|
| 6678 | long chg = -1, add = -1, spool_resv, gbl_resv;
|
| 6679 | struct hstate *h = hstate_inode(inode);
|
| 6680 | struct hugepage_subpool *spool = subpool_inode(inode);
|
| 6681 | struct resv_map *resv_map;
|
| 6682 | struct hugetlb_cgroup *h_cg = NULL;
|
| 6683 | long gbl_reserve, regions_needed = 0;
|
| 6684 | int err;
|
| 6685 |
|
| 6686 |
|
| 6687 | if (from > to) {
|
| 6688 | VM_WARN(1, "%s called with a negative range\n", __func__);
|
| 6689 | return -EINVAL;
|
| 6690 | }
|
| 6691 |
|
| 6692 | |
| 6693 | * vma specific semaphore used for pmd sharing and fault/truncation
|
| 6694 | * synchronization
|
| 6695 |
|
| 6696 | hugetlb_vma_lock_alloc(vma);
|
| 6697 |
|
| 6698 | |
| 6699 | * Only apply hugepage reservation if asked. At fault time, an
|
| 6700 | * attempt will be made for VM_NORESERVE to allocate a page
|
| 6701 | * without using reserves
|
| 6702 |
|
| 6703 | if (vma_flags_test(&vma_flags, VMA_NORESERVE_BIT))
|
| 6704 | return 0;
|
| 6705 |
|
| 6706 | |
| 6707 | * Shared mappings base their reservation on the number of pages that
|
| 6708 | * are already allocated on behalf of the file. Private mappings need
|
| 6709 | * to reserve the full area even if read-only as mprotect() may be
|
| 6710 | * called to make the mapping read-write. Assume !vma is a shm mapping
|
| 6711 |
|
| 6712 | if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
|
| 6713 | |
| 6714 | * resv_map can not be NULL as hugetlb_reserve_pages is only
|
| 6715 | * called for inodes for which resv_maps were created (see
|
| 6716 | * hugetlbfs_get_inode).
|
| 6717 |
|
| 6718 | resv_map = inode_resv_map(inode);
|
| 6719 |
|
| 6720 | chg = region_chg(resv_map, from, to, ®ions_needed);
|
| 6721 | } else {
|
| 6722 |
|
| 6723 | resv_map = resv_map_alloc();
|
| 6724 | if (!resv_map) {
|
| 6725 | err = -ENOMEM;
|
| 6726 | goto out_err;
|
| 6727 | }
|
| 6728 |
|
| 6729 | chg = to - from;
|
| 6730 |
|
| 6731 | set_vma_resv_map(vma, resv_map);
|
| 6732 | set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
|
| 6733 | }
|
| 6734 |
|
| 6735 | if (chg < 0) {
|
| 6736 |
|
| 6737 | err = (chg == -ENOMEM) ? -ENOMEM : -EINVAL;
|
| 6738 | goto out_err;
|
| 6739 | }
|
| 6740 |
|
| 6741 | err = hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
|
| 6742 | chg * pages_per_huge_page(h), &h_cg);
|
| 6743 | if (err < 0)
|
| 6744 | goto out_err;
|
| 6745 |
|
| 6746 | if (vma && !vma_test(vma, VMA_MAYSHARE_BIT) && h_cg) {
|
| 6747 | |
| 6748 | * of the resv_map.
|
| 6749 |
|
| 6750 | resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
|
| 6751 | }
|
| 6752 |
|
| 6753 | |
| 6754 | * There must be enough pages in the subpool for the mapping. If
|
| 6755 | * the subpool has a minimum size, there may be some global
|
| 6756 | * reservations already in place (gbl_reserve).
|
| 6757 |
|
| 6758 | gbl_reserve = hugepage_subpool_get_pages(spool, chg);
|
| 6759 | if (gbl_reserve < 0) {
|
| 6760 | err = gbl_reserve;
|
| 6761 | goto out_uncharge_cgroup;
|
| 6762 | }
|
| 6763 |
|
| 6764 | |
| 6765 | * Check enough hugepages are available for the reservation.
|
| 6766 | * Hand the pages back to the subpool if there are not
|
| 6767 |
|
| 6768 | err = hugetlb_acct_memory(h, gbl_reserve);
|
| 6769 | if (err < 0)
|
| 6770 | goto out_put_pages;
|
| 6771 |
|
| 6772 | |
| 6773 | * Account for the reservations made. Shared mappings record regions
|
| 6774 | * that have reservations as they are shared by multiple VMAs.
|
| 6775 | * When the last VMA disappears, the region map says how much
|
| 6776 | * the reservation was and the page cache tells how much of
|
| 6777 | * the reservation was consumed. Private mappings are per-VMA and
|
| 6778 | * only the consumed reservations are tracked. When the VMA
|
| 6779 | * disappears, the original reservation is the VMA size and the
|
| 6780 | * consumed reservations are stored in the map. Hence, nothing
|
| 6781 | * else has to be done for private mappings here
|
| 6782 |
|
| 6783 | if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
|
| 6784 | add = region_add(resv_map, from, to, regions_needed, h, h_cg);
|
| 6785 |
|
| 6786 | if (unlikely(add < 0)) {
|
| 6787 | hugetlb_acct_memory(h, -gbl_reserve);
|
| 6788 | err = add;
|
| 6789 | goto out_put_pages;
|
| 6790 | } else if (unlikely(chg > add)) {
|
| 6791 | |
| 6792 | * pages in this range were added to the reserve
|
| 6793 | * map between region_chg and region_add. This
|
| 6794 | * indicates a race with alloc_hugetlb_folio. Adjust
|
| 6795 | * the subpool and reserve counts modified above
|
| 6796 | * based on the difference.
|
| 6797 |
|
| 6798 | long rsv_adjust;
|
| 6799 |
|
| 6800 | |
| 6801 | * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
|
| 6802 | * reference to h_cg->css. See comment below for detail.
|
| 6803 |
|
| 6804 | hugetlb_cgroup_uncharge_cgroup_rsvd(
|
| 6805 | hstate_index(h),
|
| 6806 | (chg - add) * pages_per_huge_page(h), h_cg);
|
| 6807 |
|
| 6808 | rsv_adjust = hugepage_subpool_put_pages(spool,
|
| 6809 | chg - add);
|
| 6810 | hugetlb_acct_memory(h, -rsv_adjust);
|
| 6811 | } else if (h_cg) {
|
| 6812 | |
| 6813 | * The file_regions will hold their own reference to
|
| 6814 | * h_cg->css. So we should release the reference held
|
| 6815 | * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
|
| 6816 | * done.
|
| 6817 |
|
| 6818 | hugetlb_cgroup_put_rsvd_cgroup(h_cg);
|
| 6819 | }
|
| 6820 | }
|
| 6821 | return chg;
|
| 6822 |
|
| 6823 | out_put_pages:
|
| 6824 | spool_resv = chg - gbl_reserve;
|
| 6825 | if (spool_resv) {
|
| 6826 |
|
| 6827 | gbl_resv = hugepage_subpool_put_pages(spool, spool_resv);
|
| 6828 | |
| 6829 | * subpool's reserved pages can not be put back due to race,
|
| 6830 | * return to hstate.
|
| 6831 |
|
| 6832 | hugetlb_acct_memory(h, -gbl_resv);
|
| 6833 | }
|
| 6834 |
|
| 6835 | if (gbl_reserve && spool) {
|
| 6836 | unsigned long flags;
|
| 6837 |
|
| 6838 | spin_lock_irqsave(&spool->lock, flags);
|
| 6839 | if (spool->max_hpages != -1)
|
| 6840 | spool->used_hpages -= gbl_reserve;
|
| 6841 | unlock_or_release_subpool(spool, flags);
|
| 6842 | }
|
| 6843 | out_uncharge_cgroup:
|
| 6844 | hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
|
| 6845 | chg * pages_per_huge_page(h), h_cg);
|
| 6846 | out_err:
|
| 6847 | hugetlb_vma_lock_free(vma);
|
| 6848 | if (!vma || vma_test(vma, VMA_MAYSHARE_BIT))
|
| 6849 | |
| 6850 | * region_add failed or didn't run.
|
| 6851 |
|
| 6852 | if (chg >= 0 && add < 0)
|
| 6853 | region_abort(resv_map, from, to, regions_needed);
|
| 6854 | if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
|
| 6855 | kref_put(&resv_map->refs, resv_map_release);
|
| 6856 | set_vma_resv_map(vma, NULL);
|
| 6857 | }
|
| 6858 | return err;
|
| 6859 | }
|
| 6860 |
|
| 6861 | long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
|
| 6862 | long freed)
|
| 6863 | {
|
| 6864 | struct hstate *h = hstate_inode(inode);
|
| 6865 | struct resv_map *resv_map = inode_resv_map(inode);
|
| 6866 | long chg = 0;
|
| 6867 | struct hugepage_subpool *spool = subpool_inode(inode);
|
| 6868 | long gbl_reserve;
|
| 6869 |
|
| 6870 | |
| 6871 | * Since this routine can be called in the evict inode path for all
|
| 6872 | * hugetlbfs inodes, resv_map could be NULL.
|
| 6873 |
|
| 6874 | if (resv_map) {
|
| 6875 | chg = region_del(resv_map, start, end);
|
| 6876 | |
| 6877 | * region_del() can fail in the rare case where a region
|
| 6878 | * must be split and another region descriptor can not be
|
| 6879 | * allocated. If end == LONG_MAX, it will not fail.
|
| 6880 |
|
| 6881 | if (chg < 0)
|
| 6882 | return chg;
|
| 6883 | }
|
| 6884 |
|
| 6885 | spin_lock(&inode->i_lock);
|
| 6886 | inode->i_blocks -= (blocks_per_huge_page(h) * freed);
|
| 6887 | spin_unlock(&inode->i_lock);
|
| 6888 |
|
| 6889 | |
| 6890 | * If the subpool has a minimum size, the number of global
|
| 6891 | * reservations to be released may be adjusted.
|
| 6892 | *
|
| 6893 | * Note that !resv_map implies freed == 0. So (chg - freed)
|
| 6894 | * won't go negative.
|
| 6895 |
|
| 6896 | gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
|
| 6897 | hugetlb_acct_memory(h, -gbl_reserve);
|
| 6898 |
|
| 6899 | return 0;
|
| 6900 | }
|
| 6901 |
|
| 6902 | #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
|
| 6903 | static unsigned long page_table_shareable(struct vm_area_struct *svma,
|
| 6904 | struct vm_area_struct *vma,
|
| 6905 | unsigned long addr, pgoff_t idx)
|
| 6906 | {
|
| 6907 | unsigned long saddr = ((idx - vma_start_pgoff(svma)) << PAGE_SHIFT) +
|
| 6908 | svma->vm_start;
|
| 6909 | unsigned long sbase = saddr & PUD_MASK;
|
| 6910 | unsigned long s_end = sbase + PUD_SIZE;
|
| 6911 |
|
| 6912 |
|
| 6913 | vm_flags_t vm_flags = vma->vm_flags & ~VM_LOCKED_MASK;
|
| 6914 | vm_flags_t svm_flags = svma->vm_flags & ~VM_LOCKED_MASK;
|
| 6915 |
|
| 6916 | |
| 6917 | * match the virtual addresses, permission and the alignment of the
|
| 6918 | * page table page.
|
| 6919 | *
|
| 6920 | * Also, vma_lock (vm_private_data) is required for sharing.
|
| 6921 |
|
| 6922 | if (pmd_index(addr) != pmd_index(saddr) ||
|
| 6923 | vm_flags != svm_flags ||
|
| 6924 | !range_in_vma(svma, sbase, s_end) ||
|
| 6925 | !svma->vm_private_data)
|
| 6926 | return 0;
|
| 6927 |
|
| 6928 | return saddr;
|
| 6929 | }
|
| 6930 |
|
| 6931 | bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
|
| 6932 | {
|
| 6933 | unsigned long start = addr & PUD_MASK;
|
| 6934 | unsigned long end = start + PUD_SIZE;
|
| 6935 |
|
| 6936 | #ifdef CONFIG_USERFAULTFD
|
| 6937 | if (uffd_disable_huge_pmd_share(vma))
|
| 6938 | return false;
|
| 6939 | #endif
|
| 6940 | |
| 6941 | * check on proper vm_flags and page table alignment
|
| 6942 |
|
| 6943 | if (!(vma->vm_flags & VM_MAYSHARE))
|
| 6944 | return false;
|
| 6945 | if (!vma->vm_private_data)
|
| 6946 | return false;
|
| 6947 | if (!range_in_vma(vma, start, end))
|
| 6948 | return false;
|
| 6949 | return true;
|
| 6950 | }
|
| 6951 |
|
| 6952 | |
| 6953 | * Determine if start,end range within vma could be mapped by shared pmd.
|
| 6954 | * If yes, adjust start and end to cover range associated with possible
|
| 6955 | * shared pmd mappings.
|
| 6956 |
|
| 6957 | void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
|
| 6958 | unsigned long *start, unsigned long *end)
|
| 6959 | {
|
| 6960 | unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
|
| 6961 | v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
|
| 6962 |
|
| 6963 | |
| 6964 | * vma needs to span at least one aligned PUD size, and the range
|
| 6965 | * must be at least partially within in.
|
| 6966 |
|
| 6967 | if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
|
| 6968 | (*end <= v_start) || (*start >= v_end))
|
| 6969 | return;
|
| 6970 |
|
| 6971 |
|
| 6972 | if (*start > v_start)
|
| 6973 | *start = ALIGN_DOWN(*start, PUD_SIZE);
|
| 6974 |
|
| 6975 | if (*end < v_end)
|
| 6976 | *end = ALIGN(*end, PUD_SIZE);
|
| 6977 | }
|
| 6978 |
|
| 6979 | |
| 6980 | * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
|
| 6981 | * and returns the corresponding pte. While this is not necessary for the
|
| 6982 | * !shared pmd case because we can allocate the pmd later as well, it makes the
|
| 6983 | * code much cleaner. pmd allocation is essential for the shared case because
|
| 6984 | * pud has to be populated inside the same i_mmap_rwsem section - otherwise
|
| 6985 | * racing tasks could either miss the sharing (see huge_pte_offset) or select a
|
| 6986 | * bad pmd for sharing.
|
| 6987 |
|
| 6988 | pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
|
| 6989 | unsigned long addr, pud_t *pud)
|
| 6990 | {
|
| 6991 | struct address_space *mapping = vma->vm_file->f_mapping;
|
| 6992 | const pgoff_t idx = linear_page_index(vma, addr);
|
| 6993 | struct vm_area_struct *svma;
|
| 6994 | unsigned long saddr;
|
| 6995 | pte_t *spte = NULL;
|
| 6996 | pte_t *pte;
|
| 6997 |
|
| 6998 | i_mmap_lock_read(mapping);
|
| 6999 | mapping_rmap_tree_foreach(svma, mapping, idx, idx) {
|
| 7000 | if (svma == vma)
|
| 7001 | continue;
|
| 7002 |
|
| 7003 | saddr = page_table_shareable(svma, vma, addr, idx);
|
| 7004 | if (saddr) {
|
| 7005 | spte = hugetlb_walk(svma, saddr,
|
| 7006 | vma_mmu_pagesize(svma));
|
| 7007 | if (spte) {
|
| 7008 | ptdesc_pmd_pts_inc(virt_to_ptdesc(spte));
|
| 7009 | break;
|
| 7010 | }
|
| 7011 | }
|
| 7012 | }
|
| 7013 |
|
| 7014 | if (!spte)
|
| 7015 | goto out;
|
| 7016 |
|
| 7017 | spin_lock(&mm->page_table_lock);
|
| 7018 | if (pud_none(*pud)) {
|
| 7019 | pud_populate(mm, pud,
|
| 7020 | (pmd_t *)((unsigned long)spte & PAGE_MASK));
|
| 7021 | mm_inc_nr_pmds(mm);
|
| 7022 | } else {
|
| 7023 | ptdesc_pmd_pts_dec(virt_to_ptdesc(spte));
|
| 7024 | }
|
| 7025 | spin_unlock(&mm->page_table_lock);
|
| 7026 | out:
|
| 7027 | pte = (pte_t *)pmd_alloc(mm, pud, addr);
|
| 7028 | i_mmap_unlock_read(mapping);
|
| 7029 | return pte;
|
| 7030 | }
|
| 7031 |
|
| 7032 | static int __huge_pmd_unshare(struct mmu_gather *tlb,
|
| 7033 | struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
|
| 7034 | bool check_locks)
|
| 7035 | {
|
| 7036 | unsigned long sz = huge_page_size(hstate_vma(vma));
|
| 7037 | struct mm_struct *mm = vma->vm_mm;
|
| 7038 | pgd_t *pgd = pgd_offset(mm, addr);
|
| 7039 | p4d_t *p4d = p4d_offset(pgd, addr);
|
| 7040 | pud_t *pud = pud_offset(p4d, addr);
|
| 7041 |
|
| 7042 | if (sz != PMD_SIZE)
|
| 7043 | return 0;
|
| 7044 | if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep)))
|
| 7045 | return 0;
|
| 7046 | i_mmap_assert_write_locked(vma->vm_file->f_mapping);
|
| 7047 | if (check_locks)
|
| 7048 | hugetlb_vma_assert_locked(vma);
|
| 7049 | pud_clear(pud);
|
| 7050 |
|
| 7051 | tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
|
| 7052 |
|
| 7053 | mm_dec_nr_pmds(mm);
|
| 7054 | return 1;
|
| 7055 | }
|
| 7056 |
|
| 7057 | |
| 7058 | * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users
|
| 7059 | * @tlb: the current mmu_gather.
|
| 7060 | * @vma: the vma covering the pmd table.
|
| 7061 | * @addr: the address we are trying to unshare.
|
| 7062 | * @ptep: pointer into the (pmd) page table.
|
| 7063 | *
|
| 7064 | * Called with the page table lock held, the i_mmap_rwsem held in write mode
|
| 7065 | * and the hugetlb vma lock held in write mode.
|
| 7066 | *
|
| 7067 | * Note: The caller must call huge_pmd_unshare_flush() before dropping the
|
| 7068 | * i_mmap_rwsem.
|
| 7069 | *
|
| 7070 | * Returns: 1 if it was a shared PMD table and it got unmapped, or 0 if it
|
| 7071 | * was not a shared PMD table.
|
| 7072 |
|
| 7073 | int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
|
| 7074 | unsigned long addr, pte_t *ptep)
|
| 7075 | {
|
| 7076 | return __huge_pmd_unshare(tlb, vma, addr, ptep, true);
|
| 7077 | }
|
| 7078 |
|
| 7079 | |
| 7080 | * huge_pmd_unshare_flush - Complete a sequence of huge_pmd_unshare() calls
|
| 7081 | * @tlb: the current mmu_gather.
|
| 7082 | * @vma: the vma covering the pmd table.
|
| 7083 | *
|
| 7084 | * Perform necessary TLB flushes or IPI broadcasts to synchronize PMD table
|
| 7085 | * unsharing with concurrent page table walkers.
|
| 7086 | *
|
| 7087 | * This function must be called after a sequence of huge_pmd_unshare()
|
| 7088 | * calls while still holding the i_mmap_rwsem.
|
| 7089 |
|
| 7090 | void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma)
|
| 7091 | {
|
| 7092 | |
| 7093 | * We must synchronize page table unsharing such that nobody will
|
| 7094 | * try reusing a previously-shared page table while it might still
|
| 7095 | * be in use by previous sharers (TLB, GUP_fast).
|
| 7096 |
|
| 7097 | i_mmap_assert_write_locked(vma->vm_file->f_mapping);
|
| 7098 |
|
| 7099 | tlb_flush_unshared_tables(tlb);
|
| 7100 | }
|
| 7101 |
|
| 7102 | #else
|
| 7103 |
|
| 7104 | pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
|
| 7105 | unsigned long addr, pud_t *pud)
|
| 7106 | {
|
| 7107 | return NULL;
|
| 7108 | }
|
| 7109 |
|
| 7110 | static int __huge_pmd_unshare(struct mmu_gather *tlb,
|
| 7111 | struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
|
| 7112 | bool check_locks)
|
| 7113 | {
|
| 7114 | return 0;
|
| 7115 | }
|
| 7116 |
|
| 7117 | int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
|
| 7118 | unsigned long addr, pte_t *ptep)
|
| 7119 | {
|
| 7120 | return 0;
|
| 7121 | }
|
| 7122 |
|
| 7123 | void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma)
|
| 7124 | {
|
| 7125 | }
|
| 7126 |
|
| 7127 | void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
|
| 7128 | unsigned long *start, unsigned long *end)
|
| 7129 | {
|
| 7130 | }
|
| 7131 |
|
| 7132 | bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
|
| 7133 | {
|
| 7134 | return false;
|
| 7135 | }
|
| 7136 | #endif
|
| 7137 |
|
| 7138 | #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
|
| 7139 | pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
|
| 7140 | unsigned long addr, unsigned long sz)
|
| 7141 | {
|
| 7142 | pgd_t *pgd;
|
| 7143 | p4d_t *p4d;
|
| 7144 | pud_t *pud;
|
| 7145 | pte_t *pte = NULL;
|
| 7146 |
|
| 7147 | pgd = pgd_offset(mm, addr);
|
| 7148 | p4d = p4d_alloc(mm, pgd, addr);
|
| 7149 | if (!p4d)
|
| 7150 | return NULL;
|
| 7151 | pud = pud_alloc(mm, p4d, addr);
|
| 7152 | if (pud) {
|
| 7153 | if (sz == PUD_SIZE) {
|
| 7154 | pte = (pte_t *)pud;
|
| 7155 | } else {
|
| 7156 | BUG_ON(sz != PMD_SIZE);
|
| 7157 | if (want_pmd_share(vma, addr) && pud_none(*pud))
|
| 7158 | pte = huge_pmd_share(mm, vma, addr, pud);
|
| 7159 | else
|
| 7160 | pte = (pte_t *)pmd_alloc(mm, pud, addr);
|
| 7161 | }
|
| 7162 | }
|
| 7163 |
|
| 7164 | if (pte) {
|
| 7165 | pte_t pteval = ptep_get_lockless(pte);
|
| 7166 |
|
| 7167 | BUG_ON(pte_present(pteval) && !pte_huge(pteval));
|
| 7168 | }
|
| 7169 |
|
| 7170 | return pte;
|
| 7171 | }
|
| 7172 |
|
| 7173 | |
| 7174 | * huge_pte_offset() - Walk the page table to resolve the hugepage
|
| 7175 | * entry at address @addr
|
| 7176 | *
|
| 7177 | * Return: Pointer to page table entry (PUD or PMD) for
|
| 7178 | * address @addr, or NULL if a !p*d_present() entry is encountered and the
|
| 7179 | * size @sz doesn't match the hugepage size at this level of the page
|
| 7180 | * table.
|
| 7181 |
|
| 7182 | pte_t *huge_pte_offset(struct mm_struct *mm,
|
| 7183 | unsigned long addr, unsigned long sz)
|
| 7184 | {
|
| 7185 | pgd_t *pgd;
|
| 7186 | p4d_t *p4d;
|
| 7187 | pud_t *pud;
|
| 7188 | pmd_t *pmd;
|
| 7189 |
|
| 7190 | pgd = pgd_offset(mm, addr);
|
| 7191 | if (!pgd_present(*pgd))
|
| 7192 | return NULL;
|
| 7193 | p4d = p4d_offset(pgd, addr);
|
| 7194 | if (!p4d_present(*p4d))
|
| 7195 | return NULL;
|
| 7196 |
|
| 7197 | pud = pud_offset(p4d, addr);
|
| 7198 | if (sz == PUD_SIZE)
|
| 7199 |
|
| 7200 | return (pte_t *)pud;
|
| 7201 | if (!pud_present(*pud))
|
| 7202 | return NULL;
|
| 7203 |
|
| 7204 |
|
| 7205 | pmd = pmd_offset(pud, addr);
|
| 7206 |
|
| 7207 | return (pte_t *)pmd;
|
| 7208 | }
|
| 7209 |
|
| 7210 | |
| 7211 | * Return a mask that can be used to update an address to the last huge
|
| 7212 | * page in a page table page mapping size. Used to skip non-present
|
| 7213 | * page table entries when linearly scanning address ranges. Architectures
|
| 7214 | * with unique huge page to page table relationships can define their own
|
| 7215 | * version of this routine.
|
| 7216 |
|
| 7217 | unsigned long hugetlb_mask_last_page(struct hstate *h)
|
| 7218 | {
|
| 7219 | unsigned long hp_size = huge_page_size(h);
|
| 7220 |
|
| 7221 | if (hp_size == PUD_SIZE)
|
| 7222 | return P4D_SIZE - PUD_SIZE;
|
| 7223 | else if (hp_size == PMD_SIZE)
|
| 7224 | return PUD_SIZE - PMD_SIZE;
|
| 7225 | else
|
| 7226 | return 0UL;
|
| 7227 | }
|
| 7228 |
|
| 7229 | #else
|
| 7230 |
|
| 7231 |
|
| 7232 | __weak unsigned long hugetlb_mask_last_page(struct hstate *h)
|
| 7233 | {
|
| 7234 | #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
|
| 7235 | if (huge_page_size(h) == PMD_SIZE)
|
| 7236 | return PUD_SIZE - PMD_SIZE;
|
| 7237 | #endif
|
| 7238 | return 0UL;
|
| 7239 | }
|
| 7240 |
|
| 7241 | #endif
|
| 7242 |
|
| 7243 | |
| 7244 | * folio_isolate_hugetlb - try to isolate an allocated hugetlb folio
|
| 7245 | * @folio: the folio to isolate
|
| 7246 | * @list: the list to add the folio to on success
|
| 7247 | *
|
| 7248 | * Isolate an allocated (refcount > 0) hugetlb folio, marking it as
|
| 7249 | * isolated/non-migratable, and moving it from the active list to the
|
| 7250 | * given list.
|
| 7251 | *
|
| 7252 | * Isolation will fail if @folio is not an allocated hugetlb folio, or if
|
| 7253 | * it is already isolated/non-migratable.
|
| 7254 | *
|
| 7255 | * On success, an additional folio reference is taken that must be dropped
|
| 7256 | * using folio_putback_hugetlb() to undo the isolation.
|
| 7257 | *
|
| 7258 | * Return: True if isolation worked, otherwise False.
|
| 7259 |
|
| 7260 | bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list)
|
| 7261 | {
|
| 7262 | bool ret = true;
|
| 7263 |
|
| 7264 | spin_lock_irq(&hugetlb_lock);
|
| 7265 | if (!folio_test_hugetlb(folio) ||
|
| 7266 | !folio_test_hugetlb_migratable(folio) ||
|
| 7267 | !folio_try_get(folio)) {
|
| 7268 | ret = false;
|
| 7269 | goto unlock;
|
| 7270 | }
|
| 7271 | folio_clear_hugetlb_migratable(folio);
|
| 7272 | list_move_tail(&folio->lru, list);
|
| 7273 | unlock:
|
| 7274 | spin_unlock_irq(&hugetlb_lock);
|
| 7275 | return ret;
|
| 7276 | }
|
| 7277 |
|
| 7278 | int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
|
| 7279 | {
|
| 7280 | int ret = 0;
|
| 7281 |
|
| 7282 | *hugetlb = false;
|
| 7283 | spin_lock_irq(&hugetlb_lock);
|
| 7284 | if (folio_test_hugetlb(folio)) {
|
| 7285 | *hugetlb = true;
|
| 7286 | if (folio_test_hugetlb_freed(folio))
|
| 7287 | ret = 0;
|
| 7288 | else if (folio_test_hugetlb_migratable(folio) || unpoison)
|
| 7289 | ret = folio_try_get(folio);
|
| 7290 | else
|
| 7291 | ret = -EBUSY;
|
| 7292 | }
|
| 7293 | spin_unlock_irq(&hugetlb_lock);
|
| 7294 | return ret;
|
| 7295 | }
|
| 7296 |
|
| 7297 | |
| 7298 | * folio_putback_hugetlb - unisolate a hugetlb folio
|
| 7299 | * @folio: the isolated hugetlb folio
|
| 7300 | *
|
| 7301 | * Putback/un-isolate the hugetlb folio that was previous isolated using
|
| 7302 | * folio_isolate_hugetlb(): marking it non-isolated/migratable and putting it
|
| 7303 | * back onto the active list.
|
| 7304 | *
|
| 7305 | * Will drop the additional folio reference obtained through
|
| 7306 | * folio_isolate_hugetlb().
|
| 7307 |
|
| 7308 | void folio_putback_hugetlb(struct folio *folio)
|
| 7309 | {
|
| 7310 | spin_lock_irq(&hugetlb_lock);
|
| 7311 | folio_set_hugetlb_migratable(folio);
|
| 7312 | list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist);
|
| 7313 | spin_unlock_irq(&hugetlb_lock);
|
| 7314 | folio_put(folio);
|
| 7315 | }
|
| 7316 |
|
| 7317 | void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio,
|
| 7318 | enum migrate_reason reason)
|
| 7319 | {
|
| 7320 | struct hstate *h = folio_hstate(old_folio);
|
| 7321 |
|
| 7322 | hugetlb_cgroup_migrate(old_folio, new_folio);
|
| 7323 | folio_set_owner_migrate_reason(new_folio, reason);
|
| 7324 |
|
| 7325 | |
| 7326 | * transfer temporary state of the new hugetlb folio. This is
|
| 7327 | * reverse to other transitions because the newpage is going to
|
| 7328 | * be final while the old one will be freed so it takes over
|
| 7329 | * the temporary status.
|
| 7330 | *
|
| 7331 | * Also note that we have to transfer the per-node surplus state
|
| 7332 | * here as well otherwise the global surplus count will not match
|
| 7333 | * the per-node's.
|
| 7334 |
|
| 7335 | if (folio_test_hugetlb_temporary(new_folio)) {
|
| 7336 | int old_nid = folio_nid(old_folio);
|
| 7337 | int new_nid = folio_nid(new_folio);
|
| 7338 |
|
| 7339 | folio_set_hugetlb_temporary(old_folio);
|
| 7340 | folio_clear_hugetlb_temporary(new_folio);
|
| 7341 |
|
| 7342 |
|
| 7343 | |
| 7344 | * There is no need to transfer the per-node surplus state
|
| 7345 | * when we do not cross the node.
|
| 7346 |
|
| 7347 | if (new_nid != old_nid) {
|
| 7348 | spin_lock_irq(&hugetlb_lock);
|
| 7349 | if (h->surplus_huge_pages_node[old_nid]) {
|
| 7350 | h->surplus_huge_pages_node[old_nid]--;
|
| 7351 | h->surplus_huge_pages_node[new_nid]++;
|
| 7352 | }
|
| 7353 | spin_unlock_irq(&hugetlb_lock);
|
| 7354 | }
|
| 7355 | }
|
| 7356 |
|
| 7357 | |
| 7358 | * Our old folio is isolated and has "migratable" cleared until it
|
| 7359 | * is putback. As migration succeeded, set the new folio "migratable"
|
| 7360 | * and add it to the active list.
|
| 7361 |
|
| 7362 | spin_lock_irq(&hugetlb_lock);
|
| 7363 | folio_set_hugetlb_migratable(new_folio);
|
| 7364 | list_move_tail(&new_folio->lru, &(folio_hstate(new_folio))->hugepage_activelist);
|
| 7365 | spin_unlock_irq(&hugetlb_lock);
|
| 7366 | }
|
| 7367 |
|
| 7368 | |
| 7369 | * If @take_locks is false, the caller must ensure that no concurrent page table
|
| 7370 | * access can happen (except for gup_fast() and hardware page walks).
|
| 7371 | * If @take_locks is true, we take the hugetlb VMA lock (to lock out things like
|
| 7372 | * concurrent page fault handling) and the file rmap lock.
|
| 7373 |
|
| 7374 | static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
|
| 7375 | unsigned long start,
|
| 7376 | unsigned long end,
|
| 7377 | bool take_locks)
|
| 7378 | {
|
| 7379 | struct hstate *h = hstate_vma(vma);
|
| 7380 | unsigned long sz = huge_page_size(h);
|
| 7381 | struct mm_struct *mm = vma->vm_mm;
|
| 7382 | struct mmu_notifier_range range;
|
| 7383 | struct mmu_gather tlb;
|
| 7384 | unsigned long address;
|
| 7385 | spinlock_t *ptl;
|
| 7386 | pte_t *ptep;
|
| 7387 |
|
| 7388 | if (!(vma->vm_flags & VM_MAYSHARE))
|
| 7389 | return;
|
| 7390 |
|
| 7391 | if (start >= end)
|
| 7392 | return;
|
| 7393 |
|
| 7394 | flush_cache_range(vma, start, end);
|
| 7395 | tlb_gather_mmu_vma(&tlb, vma);
|
| 7396 |
|
| 7397 | |
| 7398 | * No need to call adjust_range_if_pmd_sharing_possible(), because
|
| 7399 | * we have already done the PUD_SIZE alignment.
|
| 7400 |
|
| 7401 | mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
|
| 7402 | start, end);
|
| 7403 | mmu_notifier_invalidate_range_start(&range);
|
| 7404 | if (take_locks) {
|
| 7405 | hugetlb_vma_lock_write(vma);
|
| 7406 | i_mmap_lock_write(vma->vm_file->f_mapping);
|
| 7407 | } else {
|
| 7408 | i_mmap_assert_write_locked(vma->vm_file->f_mapping);
|
| 7409 | }
|
| 7410 | for (address = start; address < end; address += PUD_SIZE) {
|
| 7411 | ptep = hugetlb_walk(vma, address, sz);
|
| 7412 | if (!ptep)
|
| 7413 | continue;
|
| 7414 | ptl = huge_pte_lock(h, mm, ptep);
|
| 7415 | __huge_pmd_unshare(&tlb, vma, address, ptep, take_locks);
|
| 7416 | spin_unlock(ptl);
|
| 7417 | }
|
| 7418 | huge_pmd_unshare_flush(&tlb, vma);
|
| 7419 | if (take_locks) {
|
| 7420 | i_mmap_unlock_write(vma->vm_file->f_mapping);
|
| 7421 | hugetlb_vma_unlock_write(vma);
|
| 7422 | }
|
| 7423 | |
| 7424 | * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see
|
| 7425 | * Documentation/mm/mmu_notifier.rst.
|
| 7426 |
|
| 7427 | mmu_notifier_invalidate_range_end(&range);
|
| 7428 | tlb_finish_mmu(&tlb);
|
| 7429 | }
|
| 7430 |
|
| 7431 | |
| 7432 | * This function will unconditionally remove all the shared pmd pgtable entries
|
| 7433 | * within the specific vma for a hugetlbfs memory range.
|
| 7434 |
|
| 7435 | void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
|
| 7436 | {
|
| 7437 | hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE),
|
| 7438 | ALIGN_DOWN(vma->vm_end, PUD_SIZE),
|
| 7439 | true);
|
| 7440 | }
|
| 7441 |
|
| 7442 | |
| 7443 | * For hugetlb, mremap() is an odd edge case - while the VMA copying is
|
| 7444 | * performed, we permit both the old and new VMAs to reference the same
|
| 7445 | * reservation.
|
| 7446 | *
|
| 7447 | * We fix this up after the operation succeeds, or if a newly allocated VMA
|
| 7448 | * is closed as a result of a failure to allocate memory.
|
| 7449 |
|
| 7450 | void fixup_hugetlb_reservations(struct vm_area_struct *vma)
|
| 7451 | {
|
| 7452 | if (is_vm_hugetlb_page(vma))
|
| 7453 | clear_vma_resv_huge_pages(vma);
|
| 7454 | }
|