| 1 | |
| 2 | |
| 3 | * Contiguous Memory Allocator
|
| 4 | *
|
| 5 | * Copyright (c) 2010-2011 by Samsung Electronics.
|
| 6 | * Copyright IBM Corporation, 2013
|
| 7 | * Copyright LG Electronics Inc., 2014
|
| 8 | * Written by:
|
| 9 | * Marek Szyprowski <[email protected]>
|
| 10 | * Michal Nazarewicz <[email protected]>
|
| 11 | * Aneesh Kumar K.V <[email protected]>
|
| 12 | * Joonsoo Kim <[email protected]>
|
| 13 |
|
| 14 |
|
| 15 | #define pr_fmt(fmt) "cma: " fmt
|
| 16 |
|
| 17 | #define CREATE_TRACE_POINTS
|
| 18 |
|
| 19 | #include <linux/memblock.h>
|
| 20 | #include <linux/err.h>
|
| 21 | #include <linux/list.h>
|
| 22 | #include <linux/mm.h>
|
| 23 | #include <linux/sizes.h>
|
| 24 | #include <linux/slab.h>
|
| 25 | #include <linux/string.h>
|
| 26 | #include <linux/string_choices.h>
|
| 27 | #include <linux/log2.h>
|
| 28 | #include <linux/cma.h>
|
| 29 | #include <linux/highmem.h>
|
| 30 | #include <linux/io.h>
|
| 31 | #include <linux/kmemleak.h>
|
| 32 | #include <trace/events/cma.h>
|
| 33 |
|
| 34 | #include "internal.h"
|
| 35 | #include "cma.h"
|
| 36 | #include "mm_init.h"
|
| 37 |
|
| 38 | struct cma cma_areas[MAX_CMA_AREAS];
|
| 39 | unsigned int cma_area_count;
|
| 40 |
|
| 41 | phys_addr_t cma_get_base(const struct cma *cma)
|
| 42 | {
|
| 43 | WARN_ON_ONCE(cma->nranges != 1);
|
| 44 | return PFN_PHYS(cma->ranges[0].base_pfn);
|
| 45 | }
|
| 46 |
|
| 47 | unsigned long cma_get_size(const struct cma *cma)
|
| 48 | {
|
| 49 | return cma->count << PAGE_SHIFT;
|
| 50 | }
|
| 51 |
|
| 52 | const char *cma_get_name(const struct cma *cma)
|
| 53 | {
|
| 54 | return cma->name;
|
| 55 | }
|
| 56 | EXPORT_SYMBOL_GPL(cma_get_name);
|
| 57 |
|
| 58 | static unsigned long cma_bitmap_aligned_mask(const struct cma *cma,
|
| 59 | unsigned int align_order)
|
| 60 | {
|
| 61 | if (align_order <= cma->order_per_bit)
|
| 62 | return 0;
|
| 63 | return (1UL << (align_order - cma->order_per_bit)) - 1;
|
| 64 | }
|
| 65 |
|
| 66 | |
| 67 | * Find the offset of the base PFN from the specified align_order.
|
| 68 | * The value returned is represented in order_per_bits.
|
| 69 |
|
| 70 | static unsigned long cma_bitmap_aligned_offset(const struct cma *cma,
|
| 71 | const struct cma_memrange *cmr,
|
| 72 | unsigned int align_order)
|
| 73 | {
|
| 74 | return (cmr->base_pfn & ((1UL << align_order) - 1))
|
| 75 | >> cma->order_per_bit;
|
| 76 | }
|
| 77 |
|
| 78 | static unsigned long cma_bitmap_pages_to_bits(const struct cma *cma,
|
| 79 | unsigned long pages)
|
| 80 | {
|
| 81 | return ALIGN(pages, 1UL << cma->order_per_bit) >> cma->order_per_bit;
|
| 82 | }
|
| 83 |
|
| 84 | static void cma_clear_bitmap(struct cma *cma, const struct cma_memrange *cmr,
|
| 85 | unsigned long pfn, unsigned long count)
|
| 86 | {
|
| 87 | unsigned long bitmap_no, bitmap_count;
|
| 88 | unsigned long flags;
|
| 89 |
|
| 90 | bitmap_no = (pfn - cmr->base_pfn) >> cma->order_per_bit;
|
| 91 | bitmap_count = cma_bitmap_pages_to_bits(cma, count);
|
| 92 |
|
| 93 | spin_lock_irqsave(&cma->lock, flags);
|
| 94 | bitmap_clear(cmr->bitmap, bitmap_no, bitmap_count);
|
| 95 | cma->available_count += count;
|
| 96 | spin_unlock_irqrestore(&cma->lock, flags);
|
| 97 | }
|
| 98 |
|
| 99 | |
| 100 | * Check if a CMA area contains no ranges that intersect with
|
| 101 | * multiple zones. Store the result in the flags in case
|
| 102 | * this gets called more than once.
|
| 103 |
|
| 104 | bool cma_validate_zones(struct cma *cma)
|
| 105 | {
|
| 106 | int r;
|
| 107 | unsigned long base_pfn;
|
| 108 | struct cma_memrange *cmr;
|
| 109 | bool valid_bit_set;
|
| 110 |
|
| 111 | |
| 112 | * If already validated, return result of previous check.
|
| 113 | * Either the valid or invalid bit will be set if this
|
| 114 | * check has already been done. If neither is set, the
|
| 115 | * check has not been performed yet.
|
| 116 |
|
| 117 | valid_bit_set = test_bit(CMA_ZONES_VALID, &cma->flags);
|
| 118 | if (valid_bit_set || test_bit(CMA_ZONES_INVALID, &cma->flags))
|
| 119 | return valid_bit_set;
|
| 120 |
|
| 121 | for (r = 0; r < cma->nranges; r++) {
|
| 122 | cmr = &cma->ranges[r];
|
| 123 | base_pfn = cmr->base_pfn;
|
| 124 |
|
| 125 | |
| 126 | * alloc_contig_range() requires the pfn range specified
|
| 127 | * to be in the same zone. Simplify by forcing the entire
|
| 128 | * CMA resv range to be in the same zone.
|
| 129 |
|
| 130 | if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) {
|
| 131 | set_bit(CMA_ZONES_INVALID, &cma->flags);
|
| 132 | return false;
|
| 133 | }
|
| 134 | }
|
| 135 |
|
| 136 | set_bit(CMA_ZONES_VALID, &cma->flags);
|
| 137 |
|
| 138 | return true;
|
| 139 | }
|
| 140 |
|
| 141 | static void __init cma_activate_area(struct cma *cma)
|
| 142 | {
|
| 143 | unsigned long pfn, end_pfn, early_pfn[CMA_MAX_RANGES];
|
| 144 | int allocrange, r;
|
| 145 | struct cma_memrange *cmr;
|
| 146 | unsigned long bitmap_count, count;
|
| 147 |
|
| 148 | for (allocrange = 0; allocrange < cma->nranges; allocrange++) {
|
| 149 | cmr = &cma->ranges[allocrange];
|
| 150 | early_pfn[allocrange] = cmr->early_pfn;
|
| 151 | cmr->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma, cmr),
|
| 152 | GFP_KERNEL);
|
| 153 | if (!cmr->bitmap)
|
| 154 | goto cleanup;
|
| 155 | }
|
| 156 |
|
| 157 | if (!cma_validate_zones(cma))
|
| 158 | goto cleanup;
|
| 159 |
|
| 160 | for (r = 0; r < cma->nranges; r++) {
|
| 161 | cmr = &cma->ranges[r];
|
| 162 | if (early_pfn[r] != cmr->base_pfn) {
|
| 163 | count = early_pfn[r] - cmr->base_pfn;
|
| 164 | bitmap_count = cma_bitmap_pages_to_bits(cma, count);
|
| 165 | bitmap_set(cmr->bitmap, 0, bitmap_count);
|
| 166 | }
|
| 167 |
|
| 168 | WARN_ON_ONCE(!pfn_valid(cmr->base_pfn));
|
| 169 |
|
| 170 | for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count;
|
| 171 | pfn += pageblock_nr_pages)
|
| 172 | init_cma_reserved_pageblock(pfn_to_page(pfn));
|
| 173 | }
|
| 174 |
|
| 175 | spin_lock_init(&cma->lock);
|
| 176 |
|
| 177 | mutex_init(&cma->alloc_mutex);
|
| 178 |
|
| 179 | #ifdef CONFIG_CMA_DEBUGFS
|
| 180 | INIT_HLIST_HEAD(&cma->mem_head);
|
| 181 | spin_lock_init(&cma->mem_head_lock);
|
| 182 | #endif
|
| 183 | set_bit(CMA_ACTIVATED, &cma->flags);
|
| 184 |
|
| 185 | return;
|
| 186 |
|
| 187 | cleanup:
|
| 188 | for (r = 0; r < allocrange; r++)
|
| 189 | bitmap_free(cma->ranges[r].bitmap);
|
| 190 |
|
| 191 |
|
| 192 | if (!test_bit(CMA_RESERVE_PAGES_ON_ERROR, &cma->flags)) {
|
| 193 | for (r = 0; r < cma->nranges; r++) {
|
| 194 | unsigned long start_pfn;
|
| 195 |
|
| 196 | cmr = &cma->ranges[r];
|
| 197 | start_pfn = r <= allocrange ? early_pfn[r] : cmr->early_pfn;
|
| 198 | end_pfn = cmr->base_pfn + cmr->count;
|
| 199 | for (pfn = start_pfn; pfn < end_pfn; pfn++)
|
| 200 | free_reserved_page(pfn_to_page(pfn));
|
| 201 | }
|
| 202 | }
|
| 203 | totalcma_pages -= cma->count;
|
| 204 | cma->available_count = cma->count = 0;
|
| 205 | pr_err("CMA area %s could not be activated\n", cma->name);
|
| 206 | }
|
| 207 |
|
| 208 | static int __init cma_init_reserved_areas(void)
|
| 209 | {
|
| 210 | int i;
|
| 211 |
|
| 212 | for (i = 0; i < cma_area_count; i++)
|
| 213 | cma_activate_area(&cma_areas[i]);
|
| 214 |
|
| 215 | return 0;
|
| 216 | }
|
| 217 | core_initcall(cma_init_reserved_areas);
|
| 218 |
|
| 219 | void __init cma_reserve_pages_on_error(struct cma *cma)
|
| 220 | {
|
| 221 | set_bit(CMA_RESERVE_PAGES_ON_ERROR, &cma->flags);
|
| 222 | }
|
| 223 |
|
| 224 | static int __init cma_new_area(const char *name, phys_addr_t size,
|
| 225 | unsigned int order_per_bit,
|
| 226 | struct cma **res_cma)
|
| 227 | {
|
| 228 | struct cma *cma;
|
| 229 |
|
| 230 | if (cma_area_count == ARRAY_SIZE(cma_areas)) {
|
| 231 | pr_err("Not enough slots for CMA reserved regions!\n");
|
| 232 | return -ENOSPC;
|
| 233 | }
|
| 234 |
|
| 235 | |
| 236 | * Each reserved area must be initialised later, when more kernel
|
| 237 | * subsystems (like slab allocator) are available.
|
| 238 |
|
| 239 | cma = &cma_areas[cma_area_count];
|
| 240 | cma_area_count++;
|
| 241 |
|
| 242 | if (name)
|
| 243 | strscpy(cma->name, name);
|
| 244 | else
|
| 245 | snprintf(cma->name, CMA_MAX_NAME, "cma%d", cma_area_count);
|
| 246 |
|
| 247 | cma->available_count = cma->count = size >> PAGE_SHIFT;
|
| 248 | cma->order_per_bit = order_per_bit;
|
| 249 | *res_cma = cma;
|
| 250 | totalcma_pages += cma->count;
|
| 251 |
|
| 252 | return 0;
|
| 253 | }
|
| 254 |
|
| 255 | static void __init cma_drop_area(struct cma *cma)
|
| 256 | {
|
| 257 | totalcma_pages -= cma->count;
|
| 258 | cma_area_count--;
|
| 259 | }
|
| 260 |
|
| 261 | |
| 262 | * cma_init_reserved_mem() - create custom contiguous area from reserved memory
|
| 263 | * @base: Base address of the reserved area
|
| 264 | * @size: Size of the reserved area (in bytes),
|
| 265 | * @order_per_bit: Order of pages represented by one bit on bitmap.
|
| 266 | * @name: The name of the area. If this parameter is NULL, the name of
|
| 267 | * the area will be set to "cmaN", where N is a running counter of
|
| 268 | * used areas.
|
| 269 | * @res_cma: Pointer to store the created cma region.
|
| 270 | *
|
| 271 | * This function creates custom contiguous area from already reserved memory.
|
| 272 |
|
| 273 | int __init cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
|
| 274 | unsigned int order_per_bit,
|
| 275 | const char *name,
|
| 276 | struct cma **res_cma)
|
| 277 | {
|
| 278 | struct cma *cma;
|
| 279 | int ret;
|
| 280 |
|
| 281 |
|
| 282 | if (!size || !memblock_is_region_reserved(base, size))
|
| 283 | return -EINVAL;
|
| 284 |
|
| 285 | |
| 286 | * CMA uses CMA_MIN_ALIGNMENT_BYTES as alignment requirement which
|
| 287 | * needs pageblock_order to be initialized. Let's enforce it.
|
| 288 |
|
| 289 | if (!pageblock_order) {
|
| 290 | pr_err("pageblock_order not yet initialized. Called during early boot?\n");
|
| 291 | return -EINVAL;
|
| 292 | }
|
| 293 |
|
| 294 |
|
| 295 | if (!IS_ALIGNED(base | size, CMA_MIN_ALIGNMENT_BYTES))
|
| 296 | return -EINVAL;
|
| 297 |
|
| 298 | ret = cma_new_area(name, size, order_per_bit, &cma);
|
| 299 | if (ret != 0)
|
| 300 | return ret;
|
| 301 |
|
| 302 | cma->ranges[0].base_pfn = PFN_DOWN(base);
|
| 303 | cma->ranges[0].early_pfn = PFN_DOWN(base);
|
| 304 | cma->ranges[0].count = cma->count;
|
| 305 | cma->nranges = 1;
|
| 306 | cma->nid = NUMA_NO_NODE;
|
| 307 |
|
| 308 | *res_cma = cma;
|
| 309 |
|
| 310 | return 0;
|
| 311 | }
|
| 312 |
|
| 313 | |
| 314 | * Structure used while walking physical memory ranges and finding out
|
| 315 | * which one(s) to use for a CMA area.
|
| 316 |
|
| 317 | struct cma_init_memrange {
|
| 318 | phys_addr_t base;
|
| 319 | phys_addr_t size;
|
| 320 | struct list_head list;
|
| 321 | };
|
| 322 |
|
| 323 | |
| 324 | * Work array used during CMA initialization.
|
| 325 |
|
| 326 | static struct cma_init_memrange memranges[CMA_MAX_RANGES] __initdata;
|
| 327 |
|
| 328 | static bool __init revsizecmp(struct cma_init_memrange *mlp,
|
| 329 | struct cma_init_memrange *mrp)
|
| 330 | {
|
| 331 | return mlp->size > mrp->size;
|
| 332 | }
|
| 333 |
|
| 334 | static bool __init basecmp(struct cma_init_memrange *mlp,
|
| 335 | struct cma_init_memrange *mrp)
|
| 336 | {
|
| 337 | return mlp->base < mrp->base;
|
| 338 | }
|
| 339 |
|
| 340 | |
| 341 | * Helper function to create sorted lists.
|
| 342 |
|
| 343 | static void __init list_insert_sorted(
|
| 344 | struct list_head *ranges,
|
| 345 | struct cma_init_memrange *mrp,
|
| 346 | bool (*cmp)(struct cma_init_memrange *lh, struct cma_init_memrange *rh))
|
| 347 | {
|
| 348 | struct list_head *mp;
|
| 349 | struct cma_init_memrange *mlp;
|
| 350 |
|
| 351 | if (list_empty(ranges))
|
| 352 | list_add(&mrp->list, ranges);
|
| 353 | else {
|
| 354 | list_for_each(mp, ranges) {
|
| 355 | mlp = list_entry(mp, struct cma_init_memrange, list);
|
| 356 | if (cmp(mlp, mrp))
|
| 357 | break;
|
| 358 | }
|
| 359 | __list_add(&mrp->list, mlp->list.prev, &mlp->list);
|
| 360 | }
|
| 361 | }
|
| 362 |
|
| 363 | static int __init cma_fixed_reserve(phys_addr_t base, phys_addr_t size)
|
| 364 | {
|
| 365 | if (IS_ENABLED(CONFIG_HIGHMEM)) {
|
| 366 | phys_addr_t highmem_start = __pa(high_memory - 1) + 1;
|
| 367 |
|
| 368 | |
| 369 | * If allocating at a fixed base the request region must not
|
| 370 | * cross the low/high memory boundary.
|
| 371 |
|
| 372 | if (base < highmem_start && base + size > highmem_start) {
|
| 373 | pr_err("Region at %pa defined on low/high memory boundary (%pa)\n",
|
| 374 | &base, &highmem_start);
|
| 375 | return -EINVAL;
|
| 376 | }
|
| 377 | }
|
| 378 |
|
| 379 | if (memblock_is_region_reserved(base, size) ||
|
| 380 | memblock_reserve(base, size) < 0) {
|
| 381 | return -EBUSY;
|
| 382 | }
|
| 383 |
|
| 384 | return 0;
|
| 385 | }
|
| 386 |
|
| 387 | static phys_addr_t __init cma_alloc_mem(phys_addr_t base, phys_addr_t size,
|
| 388 | phys_addr_t align, phys_addr_t limit, int nid)
|
| 389 | {
|
| 390 | phys_addr_t addr = 0;
|
| 391 |
|
| 392 | |
| 393 | * If there is enough memory, try a bottom-up allocation first.
|
| 394 | * It will place the new cma area close to the start of the node
|
| 395 | * and guarantee that the compaction is moving pages out of the
|
| 396 | * cma area and not into it.
|
| 397 | * Avoid using first 4GB to not interfere with constrained zones
|
| 398 | * like DMA/DMA32.
|
| 399 |
|
| 400 | #ifdef CONFIG_PHYS_ADDR_T_64BIT
|
| 401 | if (!memblock_bottom_up() && limit >= SZ_4G + size) {
|
| 402 | memblock_set_bottom_up(true);
|
| 403 | addr = memblock_alloc_range_nid(size, align, SZ_4G, limit,
|
| 404 | nid, true);
|
| 405 | memblock_set_bottom_up(false);
|
| 406 | }
|
| 407 | #endif
|
| 408 |
|
| 409 | |
| 410 | * On systems with HIGHMEM try allocating from there before consuming
|
| 411 | * memory in lower zones.
|
| 412 |
|
| 413 | if (!addr && IS_ENABLED(CONFIG_HIGHMEM)) {
|
| 414 | phys_addr_t highmem = __pa(high_memory - 1) + 1;
|
| 415 |
|
| 416 | |
| 417 | * All pages in the reserved area must come from the same zone.
|
| 418 | * If the requested region crosses the low/high memory boundary,
|
| 419 | * try allocating from high memory first and fall back to low
|
| 420 | * memory in case of failure.
|
| 421 |
|
| 422 | if (base < highmem && limit > highmem) {
|
| 423 | addr = memblock_alloc_range_nid(size, align, highmem,
|
| 424 | limit, nid, true);
|
| 425 | limit = highmem;
|
| 426 | }
|
| 427 | }
|
| 428 |
|
| 429 | if (!addr)
|
| 430 | addr = memblock_alloc_range_nid(size, align, base, limit, nid,
|
| 431 | true);
|
| 432 |
|
| 433 | return addr;
|
| 434 | }
|
| 435 |
|
| 436 | static int __init __cma_declare_contiguous_nid(phys_addr_t *basep,
|
| 437 | phys_addr_t size, phys_addr_t limit,
|
| 438 | phys_addr_t alignment, unsigned int order_per_bit,
|
| 439 | bool fixed, const char *name, struct cma **res_cma,
|
| 440 | int nid)
|
| 441 | {
|
| 442 | phys_addr_t memblock_end = memblock_end_of_DRAM();
|
| 443 | phys_addr_t base = *basep;
|
| 444 | int ret;
|
| 445 |
|
| 446 | pr_debug("%s(size %pa, base %pa, limit %pa alignment %pa)\n",
|
| 447 | __func__, &size, &base, &limit, &alignment);
|
| 448 |
|
| 449 | if (cma_area_count == ARRAY_SIZE(cma_areas)) {
|
| 450 | pr_err("Not enough slots for CMA reserved regions!\n");
|
| 451 | return -ENOSPC;
|
| 452 | }
|
| 453 |
|
| 454 | if (!size)
|
| 455 | return -EINVAL;
|
| 456 |
|
| 457 | if (alignment && !is_power_of_2(alignment))
|
| 458 | return -EINVAL;
|
| 459 |
|
| 460 | if (!IS_ENABLED(CONFIG_NUMA))
|
| 461 | nid = NUMA_NO_NODE;
|
| 462 |
|
| 463 |
|
| 464 | alignment = max_t(phys_addr_t, alignment, CMA_MIN_ALIGNMENT_BYTES);
|
| 465 | if (fixed && base & (alignment - 1)) {
|
| 466 | pr_err("Region at %pa must be aligned to %pa bytes\n",
|
| 467 | &base, &alignment);
|
| 468 | return -EINVAL;
|
| 469 | }
|
| 470 | base = ALIGN(base, alignment);
|
| 471 | size = ALIGN(size, alignment);
|
| 472 | limit &= ~(alignment - 1);
|
| 473 |
|
| 474 | if (!base)
|
| 475 | fixed = false;
|
| 476 |
|
| 477 |
|
| 478 | if (!IS_ALIGNED(size >> PAGE_SHIFT, 1 << order_per_bit))
|
| 479 | return -EINVAL;
|
| 480 |
|
| 481 |
|
| 482 | |
| 483 | * If the limit is unspecified or above the memblock end, its effective
|
| 484 | * value will be the memblock end. Set it explicitly to simplify further
|
| 485 | * checks.
|
| 486 |
|
| 487 | if (limit == 0 || limit > memblock_end)
|
| 488 | limit = memblock_end;
|
| 489 |
|
| 490 | if (base + size > limit) {
|
| 491 | pr_err("Size (%pa) of region at %pa exceeds limit (%pa)\n",
|
| 492 | &size, &base, &limit);
|
| 493 | return -EINVAL;
|
| 494 | }
|
| 495 |
|
| 496 |
|
| 497 | if (fixed) {
|
| 498 | ret = cma_fixed_reserve(base, size);
|
| 499 | if (ret)
|
| 500 | return ret;
|
| 501 | } else {
|
| 502 | base = cma_alloc_mem(base, size, alignment, limit, nid);
|
| 503 | if (!base)
|
| 504 | return -ENOMEM;
|
| 505 |
|
| 506 | |
| 507 | * kmemleak scans/reads tracked objects for pointers to other
|
| 508 | * objects but this address isn't mapped and accessible
|
| 509 |
|
| 510 | kmemleak_ignore_phys(base);
|
| 511 | }
|
| 512 |
|
| 513 | ret = cma_init_reserved_mem(base, size, order_per_bit, name, res_cma);
|
| 514 | if (ret) {
|
| 515 | memblock_phys_free(base, size);
|
| 516 | return ret;
|
| 517 | }
|
| 518 |
|
| 519 | (*res_cma)->nid = nid;
|
| 520 | *basep = base;
|
| 521 |
|
| 522 | return 0;
|
| 523 | }
|
| 524 |
|
| 525 | |
| 526 | * Create CMA areas with a total size of @total_size. A normal allocation
|
| 527 | * for one area is tried first. If that fails, the biggest memblock
|
| 528 | * ranges above 4G are selected, and allocated bottom up.
|
| 529 | *
|
| 530 | * The complexity here is not great, but this function will only be
|
| 531 | * called during boot, and the lists operated on have fewer than
|
| 532 | * CMA_MAX_RANGES elements (default value: 8).
|
| 533 |
|
| 534 | int __init cma_declare_contiguous_multi(phys_addr_t total_size,
|
| 535 | phys_addr_t align, unsigned int order_per_bit,
|
| 536 | const char *name, struct cma **res_cma, int nid)
|
| 537 | {
|
| 538 | phys_addr_t start = 0, end;
|
| 539 | phys_addr_t size, sizesum, sizeleft;
|
| 540 | struct cma_init_memrange *mrp, *mlp, *failed;
|
| 541 | struct cma_memrange *cmrp;
|
| 542 | LIST_HEAD(ranges);
|
| 543 | LIST_HEAD(final_ranges);
|
| 544 | struct list_head *mp, *next;
|
| 545 | int ret, nr = 1;
|
| 546 | u64 i;
|
| 547 | struct cma *cma;
|
| 548 |
|
| 549 | |
| 550 | * First, try it the normal way, producing just one range.
|
| 551 |
|
| 552 | ret = __cma_declare_contiguous_nid(&start, total_size, 0, align,
|
| 553 | order_per_bit, false, name, res_cma, nid);
|
| 554 | if (ret != -ENOMEM)
|
| 555 | goto out;
|
| 556 |
|
| 557 | |
| 558 | * Couldn't find one range that fits our needs, so try multiple
|
| 559 | * ranges.
|
| 560 | *
|
| 561 | * No need to do the alignment checks here, the call to
|
| 562 | * cma_declare_contiguous_nid above would have caught
|
| 563 | * any issues. With the checks, we know that:
|
| 564 | *
|
| 565 | * - @align is a power of 2
|
| 566 | * - @align is >= pageblock alignment
|
| 567 | * - @size is aligned to @align and to @order_per_bit
|
| 568 | *
|
| 569 | * So, as long as we create ranges that have a base
|
| 570 | * aligned to @align, and a size that is aligned to
|
| 571 | * both @align and @order_to_bit, things will work out.
|
| 572 |
|
| 573 | nr = 0;
|
| 574 | sizesum = 0;
|
| 575 | failed = NULL;
|
| 576 |
|
| 577 | ret = cma_new_area(name, total_size, order_per_bit, &cma);
|
| 578 | if (ret != 0)
|
| 579 | goto out;
|
| 580 |
|
| 581 | align = max_t(phys_addr_t, align, CMA_MIN_ALIGNMENT_BYTES);
|
| 582 | |
| 583 | * Create a list of ranges above 4G, largest range first.
|
| 584 |
|
| 585 | for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &start, &end, NULL) {
|
| 586 | if (upper_32_bits(start) == 0)
|
| 587 | continue;
|
| 588 |
|
| 589 | start = ALIGN(start, align);
|
| 590 | if (start >= end)
|
| 591 | continue;
|
| 592 |
|
| 593 | end = ALIGN_DOWN(end, align);
|
| 594 | if (end <= start)
|
| 595 | continue;
|
| 596 |
|
| 597 | size = end - start;
|
| 598 | size = ALIGN_DOWN(size, (PAGE_SIZE << order_per_bit));
|
| 599 | if (!size)
|
| 600 | continue;
|
| 601 | sizesum += size;
|
| 602 |
|
| 603 | pr_debug("consider %016llx - %016llx\n", (u64)start, (u64)end);
|
| 604 |
|
| 605 | |
| 606 | * If we don't yet have used the maximum number of
|
| 607 | * areas, grab a new one.
|
| 608 | *
|
| 609 | * If we can't use anymore, see if this range is not
|
| 610 | * smaller than the smallest one already recorded. If
|
| 611 | * not, re-use the smallest element.
|
| 612 |
|
| 613 | if (nr < CMA_MAX_RANGES)
|
| 614 | mrp = &memranges[nr++];
|
| 615 | else {
|
| 616 | mrp = list_last_entry(&ranges,
|
| 617 | struct cma_init_memrange, list);
|
| 618 | if (size < mrp->size)
|
| 619 | continue;
|
| 620 | list_del(&mrp->list);
|
| 621 | sizesum -= mrp->size;
|
| 622 | pr_debug("deleted %016llx - %016llx from the list\n",
|
| 623 | (u64)mrp->base, (u64)mrp->base + size);
|
| 624 | }
|
| 625 | mrp->base = start;
|
| 626 | mrp->size = size;
|
| 627 |
|
| 628 | |
| 629 | * Now do a sorted insert.
|
| 630 |
|
| 631 | list_insert_sorted(&ranges, mrp, revsizecmp);
|
| 632 | pr_debug("added %016llx - %016llx to the list\n",
|
| 633 | (u64)mrp->base, (u64)mrp->base + size);
|
| 634 | pr_debug("total size now %llu\n", (u64)sizesum);
|
| 635 | }
|
| 636 |
|
| 637 | |
| 638 | * There is not enough room in the CMA_MAX_RANGES largest
|
| 639 | * ranges, so bail out.
|
| 640 |
|
| 641 | if (sizesum < total_size) {
|
| 642 | cma_drop_area(cma);
|
| 643 | ret = -ENOMEM;
|
| 644 | goto out;
|
| 645 | }
|
| 646 |
|
| 647 | |
| 648 | * Found ranges that provide enough combined space.
|
| 649 | * Now, sorted them by address, smallest first, because we
|
| 650 | * want to mimic a bottom-up memblock allocation.
|
| 651 |
|
| 652 | sizesum = 0;
|
| 653 | list_for_each_safe(mp, next, &ranges) {
|
| 654 | mlp = list_entry(mp, struct cma_init_memrange, list);
|
| 655 | list_del(mp);
|
| 656 | list_insert_sorted(&final_ranges, mlp, basecmp);
|
| 657 | sizesum += mlp->size;
|
| 658 | if (sizesum >= total_size)
|
| 659 | break;
|
| 660 | }
|
| 661 |
|
| 662 | |
| 663 | * Walk the final list, and add a CMA range for
|
| 664 | * each range, possibly not using the last one fully.
|
| 665 |
|
| 666 | nr = 0;
|
| 667 | sizeleft = total_size;
|
| 668 | list_for_each(mp, &final_ranges) {
|
| 669 | mlp = list_entry(mp, struct cma_init_memrange, list);
|
| 670 | size = min(sizeleft, mlp->size);
|
| 671 | if (memblock_reserve(mlp->base, size)) {
|
| 672 | |
| 673 | * Unexpected error. Could go on to
|
| 674 | * the next one, but just abort to
|
| 675 | * be safe.
|
| 676 |
|
| 677 | failed = mlp;
|
| 678 | break;
|
| 679 | }
|
| 680 |
|
| 681 | pr_debug("created region %d: %016llx - %016llx\n",
|
| 682 | nr, (u64)mlp->base, (u64)mlp->base + size);
|
| 683 | cmrp = &cma->ranges[nr++];
|
| 684 | cmrp->base_pfn = PHYS_PFN(mlp->base);
|
| 685 | cmrp->early_pfn = cmrp->base_pfn;
|
| 686 | cmrp->count = size >> PAGE_SHIFT;
|
| 687 |
|
| 688 | sizeleft -= size;
|
| 689 | if (sizeleft == 0)
|
| 690 | break;
|
| 691 | }
|
| 692 |
|
| 693 | if (failed) {
|
| 694 | list_for_each(mp, &final_ranges) {
|
| 695 | mlp = list_entry(mp, struct cma_init_memrange, list);
|
| 696 | if (mlp == failed)
|
| 697 | break;
|
| 698 | memblock_phys_free(mlp->base, mlp->size);
|
| 699 | }
|
| 700 | cma_drop_area(cma);
|
| 701 | ret = -ENOMEM;
|
| 702 | goto out;
|
| 703 | }
|
| 704 |
|
| 705 | cma->nranges = nr;
|
| 706 | cma->nid = nid;
|
| 707 | *res_cma = cma;
|
| 708 |
|
| 709 | out:
|
| 710 | if (ret != 0)
|
| 711 | pr_err("Failed to reserve %lu MiB\n",
|
| 712 | (unsigned long)total_size / SZ_1M);
|
| 713 | else
|
| 714 | pr_info("Reserved %lu MiB in %d range%s\n",
|
| 715 | (unsigned long)total_size / SZ_1M, nr, str_plural(nr));
|
| 716 | return ret;
|
| 717 | }
|
| 718 |
|
| 719 | |
| 720 | * cma_declare_contiguous_nid() - reserve custom contiguous area
|
| 721 | * @base: Base address of the reserved area optional, use 0 for any
|
| 722 | * @size: Size of the reserved area (in bytes),
|
| 723 | * @limit: End address of the reserved memory (optional, 0 for any).
|
| 724 | * @alignment: Alignment for the CMA area, should be power of 2 or zero
|
| 725 | * @order_per_bit: Order of pages represented by one bit on bitmap.
|
| 726 | * @fixed: hint about where to place the reserved area
|
| 727 | * @name: The name of the area. See function cma_init_reserved_mem()
|
| 728 | * @res_cma: Pointer to store the created cma region.
|
| 729 | * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
|
| 730 | *
|
| 731 | * This function reserves memory from early allocator. It should be
|
| 732 | * called by arch specific code once the early allocator (memblock or bootmem)
|
| 733 | * has been activated and all other subsystems have already allocated/reserved
|
| 734 | * memory. This function allows to create custom reserved areas.
|
| 735 | *
|
| 736 | * If @fixed is true, reserve contiguous area at exactly @base. If false,
|
| 737 | * reserve in range from @base to @limit.
|
| 738 |
|
| 739 | int __init cma_declare_contiguous_nid(phys_addr_t base,
|
| 740 | phys_addr_t size, phys_addr_t limit,
|
| 741 | phys_addr_t alignment, unsigned int order_per_bit,
|
| 742 | bool fixed, const char *name, struct cma **res_cma,
|
| 743 | int nid)
|
| 744 | {
|
| 745 | int ret;
|
| 746 |
|
| 747 | ret = __cma_declare_contiguous_nid(&base, size, limit, alignment,
|
| 748 | order_per_bit, fixed, name, res_cma, nid);
|
| 749 | if (ret != 0)
|
| 750 | pr_err("Failed to reserve %ld MiB\n",
|
| 751 | (unsigned long)size / SZ_1M);
|
| 752 | else
|
| 753 | pr_info("Reserved %ld MiB at %pa\n",
|
| 754 | (unsigned long)size / SZ_1M, &base);
|
| 755 |
|
| 756 | return ret;
|
| 757 | }
|
| 758 |
|
| 759 | static void cma_debug_show_areas(struct cma *cma)
|
| 760 | {
|
| 761 | unsigned long start, end;
|
| 762 | unsigned long nr_part;
|
| 763 | unsigned long nbits;
|
| 764 | int r;
|
| 765 | struct cma_memrange *cmr;
|
| 766 |
|
| 767 | spin_lock_irq(&cma->lock);
|
| 768 | pr_info("number of available pages: ");
|
| 769 | for (r = 0; r < cma->nranges; r++) {
|
| 770 | cmr = &cma->ranges[r];
|
| 771 |
|
| 772 | nbits = cma_bitmap_maxno(cma, cmr);
|
| 773 |
|
| 774 | pr_info("range %d: ", r);
|
| 775 | for_each_clear_bitrange(start, end, cmr->bitmap, nbits) {
|
| 776 | nr_part = (end - start) << cma->order_per_bit;
|
| 777 | pr_cont("%s%lu@%lu", start ? "+" : "", nr_part, start);
|
| 778 | }
|
| 779 | pr_info("\n");
|
| 780 | }
|
| 781 | pr_cont("=> %lu free of %lu total pages\n", cma->available_count,
|
| 782 | cma->count);
|
| 783 | spin_unlock_irq(&cma->lock);
|
| 784 | }
|
| 785 |
|
| 786 | static int cma_range_alloc(struct cma *cma, struct cma_memrange *cmr,
|
| 787 | unsigned long count, unsigned int align,
|
| 788 | struct page **pagep, gfp_t gfp)
|
| 789 | {
|
| 790 | unsigned long bitmap_maxno, bitmap_no, bitmap_count;
|
| 791 | unsigned long start, pfn, mask, offset;
|
| 792 | int ret = -EBUSY;
|
| 793 | struct page *page = NULL;
|
| 794 |
|
| 795 | mask = cma_bitmap_aligned_mask(cma, align);
|
| 796 | offset = cma_bitmap_aligned_offset(cma, cmr, align);
|
| 797 | bitmap_maxno = cma_bitmap_maxno(cma, cmr);
|
| 798 | bitmap_count = cma_bitmap_pages_to_bits(cma, count);
|
| 799 |
|
| 800 | if (bitmap_count > bitmap_maxno)
|
| 801 | goto out;
|
| 802 |
|
| 803 | for (start = 0; ; start = bitmap_no + mask + 1) {
|
| 804 | spin_lock_irq(&cma->lock);
|
| 805 | |
| 806 | * If the request is larger than the available number
|
| 807 | * of pages, stop right away.
|
| 808 |
|
| 809 | if (count > cma->available_count) {
|
| 810 | spin_unlock_irq(&cma->lock);
|
| 811 | break;
|
| 812 | }
|
| 813 | bitmap_no = bitmap_find_next_zero_area_off(cmr->bitmap,
|
| 814 | bitmap_maxno, start, bitmap_count, mask,
|
| 815 | offset);
|
| 816 | if (bitmap_no >= bitmap_maxno) {
|
| 817 | spin_unlock_irq(&cma->lock);
|
| 818 | break;
|
| 819 | }
|
| 820 |
|
| 821 | pfn = cmr->base_pfn + (bitmap_no << cma->order_per_bit);
|
| 822 | page = pfn_to_page(pfn);
|
| 823 |
|
| 824 | |
| 825 | * Do not hand out page ranges that are not contiguous, so
|
| 826 | * callers can just iterate the pages without having to worry
|
| 827 | * about these corner cases.
|
| 828 |
|
| 829 | if (!page_range_contiguous(page, count)) {
|
| 830 | spin_unlock_irq(&cma->lock);
|
| 831 | pr_warn_ratelimited("%s: %s: skipping incompatible area [0x%lx-0x%lx]",
|
| 832 | __func__, cma->name, pfn, pfn + count - 1);
|
| 833 | continue;
|
| 834 | }
|
| 835 |
|
| 836 | bitmap_set(cmr->bitmap, bitmap_no, bitmap_count);
|
| 837 | cma->available_count -= count;
|
| 838 | |
| 839 | * It's safe to drop the lock here. We've marked this region for
|
| 840 | * our exclusive use. If the migration fails we will take the
|
| 841 | * lock again and unmark it.
|
| 842 |
|
| 843 | spin_unlock_irq(&cma->lock);
|
| 844 |
|
| 845 | mutex_lock(&cma->alloc_mutex);
|
| 846 | ret = alloc_contig_frozen_range(pfn, pfn + count, ACR_FLAGS_CMA, gfp);
|
| 847 | mutex_unlock(&cma->alloc_mutex);
|
| 848 | if (!ret)
|
| 849 | break;
|
| 850 |
|
| 851 | cma_clear_bitmap(cma, cmr, pfn, count);
|
| 852 | if (ret != -EBUSY)
|
| 853 | break;
|
| 854 |
|
| 855 | pr_debug("%s(): memory range at pfn 0x%lx %p is busy, retrying\n",
|
| 856 | __func__, pfn, page);
|
| 857 |
|
| 858 | trace_cma_alloc_busy_retry(cma->name, pfn, page, count, align);
|
| 859 | }
|
| 860 | out:
|
| 861 | if (!ret)
|
| 862 | *pagep = page;
|
| 863 | return ret;
|
| 864 | }
|
| 865 |
|
| 866 | static struct page *__cma_alloc_frozen(struct cma *cma,
|
| 867 | unsigned long count, unsigned int align, gfp_t gfp)
|
| 868 | {
|
| 869 | struct page *page = NULL;
|
| 870 | int ret = -ENOMEM, r;
|
| 871 | unsigned long i;
|
| 872 | const char *name = cma ? cma->name : NULL;
|
| 873 |
|
| 874 | if (!cma || !cma->count)
|
| 875 | return page;
|
| 876 |
|
| 877 | pr_debug("%s(cma %p, name: %s, count %lu, align %d)\n", __func__,
|
| 878 | (void *)cma, cma->name, count, align);
|
| 879 |
|
| 880 | if (!count)
|
| 881 | return page;
|
| 882 |
|
| 883 | trace_cma_alloc_start(name, count, cma->available_count, cma->count, align);
|
| 884 |
|
| 885 | for (r = 0; r < cma->nranges; r++) {
|
| 886 | page = NULL;
|
| 887 |
|
| 888 | ret = cma_range_alloc(cma, &cma->ranges[r], count, align,
|
| 889 | &page, gfp);
|
| 890 | if (ret != -EBUSY || page)
|
| 891 | break;
|
| 892 | }
|
| 893 |
|
| 894 | |
| 895 | * CMA can allocate multiple page blocks, which results in different
|
| 896 | * blocks being marked with different tags. Reset the tags to ignore
|
| 897 | * those page blocks.
|
| 898 |
|
| 899 | if (page) {
|
| 900 | for (i = 0; i < count; i++)
|
| 901 | page_kasan_tag_reset(page + i);
|
| 902 | }
|
| 903 |
|
| 904 | if (ret && !(gfp & __GFP_NOWARN)) {
|
| 905 | pr_err_ratelimited("%s: %s: alloc failed, req-size: %lu pages, ret: %d\n",
|
| 906 | __func__, cma->name, count, ret);
|
| 907 | cma_debug_show_areas(cma);
|
| 908 | }
|
| 909 |
|
| 910 | pr_debug("%s(): returned %p\n", __func__, page);
|
| 911 | trace_cma_alloc_finish(name, page ? page_to_pfn(page) : 0,
|
| 912 | page, count, align, ret);
|
| 913 | if (page) {
|
| 914 | count_vm_event(CMA_ALLOC_SUCCESS);
|
| 915 | cma_sysfs_account_success_pages(cma, count);
|
| 916 | } else {
|
| 917 | count_vm_event(CMA_ALLOC_FAIL);
|
| 918 | cma_sysfs_account_fail_pages(cma, count);
|
| 919 | }
|
| 920 |
|
| 921 | return page;
|
| 922 | }
|
| 923 |
|
| 924 | struct page *cma_alloc_frozen(struct cma *cma, unsigned long count,
|
| 925 | unsigned int align, bool no_warn)
|
| 926 | {
|
| 927 | gfp_t gfp = GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0);
|
| 928 |
|
| 929 | return __cma_alloc_frozen(cma, count, align, gfp);
|
| 930 | }
|
| 931 |
|
| 932 | struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order)
|
| 933 | {
|
| 934 | gfp_t gfp = GFP_KERNEL | __GFP_COMP | __GFP_NOWARN;
|
| 935 |
|
| 936 | return __cma_alloc_frozen(cma, 1 << order, order, gfp);
|
| 937 | }
|
| 938 |
|
| 939 | |
| 940 | * cma_alloc() - allocate pages from contiguous area
|
| 941 | * @cma: Contiguous memory region for which the allocation is performed.
|
| 942 | * @count: Requested number of pages.
|
| 943 | * @align: Requested alignment of pages (in PAGE_SIZE order).
|
| 944 | * @no_warn: Avoid printing message about failed allocation
|
| 945 | *
|
| 946 | * This function allocates part of contiguous memory on specific
|
| 947 | * contiguous memory area.
|
| 948 |
|
| 949 | struct page *cma_alloc(struct cma *cma, unsigned long count,
|
| 950 | unsigned int align, bool no_warn)
|
| 951 | {
|
| 952 | struct page *page;
|
| 953 |
|
| 954 | page = cma_alloc_frozen(cma, count, align, no_warn);
|
| 955 | if (page)
|
| 956 | set_pages_refcounted(page, count);
|
| 957 |
|
| 958 | return page;
|
| 959 | }
|
| 960 | EXPORT_SYMBOL_GPL(cma_alloc);
|
| 961 |
|
| 962 | static struct cma_memrange *find_cma_memrange(struct cma *cma,
|
| 963 | const struct page *pages, unsigned long count)
|
| 964 | {
|
| 965 | struct cma_memrange *cmr = NULL;
|
| 966 | unsigned long pfn, end_pfn;
|
| 967 | int r;
|
| 968 |
|
| 969 | pr_debug("%s(page %p, count %lu)\n", __func__, (void *)pages, count);
|
| 970 |
|
| 971 | if (!cma || !pages || count > cma->count)
|
| 972 | return NULL;
|
| 973 |
|
| 974 | pfn = page_to_pfn(pages);
|
| 975 |
|
| 976 | for (r = 0; r < cma->nranges; r++) {
|
| 977 | cmr = &cma->ranges[r];
|
| 978 | end_pfn = cmr->base_pfn + cmr->count;
|
| 979 | if (pfn >= cmr->base_pfn && pfn < end_pfn) {
|
| 980 | if (pfn + count <= end_pfn)
|
| 981 | break;
|
| 982 |
|
| 983 | VM_WARN_ON_ONCE(1);
|
| 984 | }
|
| 985 | }
|
| 986 |
|
| 987 | if (r == cma->nranges) {
|
| 988 | pr_debug("%s(page %p, count %lu, no cma range matches the page range)\n",
|
| 989 | __func__, (void *)pages, count);
|
| 990 | return NULL;
|
| 991 | }
|
| 992 |
|
| 993 | return cmr;
|
| 994 | }
|
| 995 |
|
| 996 | static void __cma_release_frozen(struct cma *cma, struct cma_memrange *cmr,
|
| 997 | const struct page *pages, unsigned long count)
|
| 998 | {
|
| 999 | unsigned long pfn = page_to_pfn(pages);
|
| 1000 |
|
| 1001 | pr_debug("%s(page %p, count %lu)\n", __func__, (void *)pages, count);
|
| 1002 |
|
| 1003 | free_contig_frozen_range(pfn, count);
|
| 1004 | cma_clear_bitmap(cma, cmr, pfn, count);
|
| 1005 | cma_sysfs_account_release_pages(cma, count);
|
| 1006 | trace_cma_release(cma->name, pfn, pages, count);
|
| 1007 | }
|
| 1008 |
|
| 1009 | |
| 1010 | * cma_release() - release allocated pages
|
| 1011 | * @cma: Contiguous memory region for which the allocation is performed.
|
| 1012 | * @pages: Allocated pages.
|
| 1013 | * @count: Number of allocated pages.
|
| 1014 | *
|
| 1015 | * This function releases memory allocated by cma_alloc().
|
| 1016 | * It returns false when provided pages do not belong to contiguous area and
|
| 1017 | * true otherwise.
|
| 1018 |
|
| 1019 | bool cma_release(struct cma *cma, const struct page *pages,
|
| 1020 | unsigned long count)
|
| 1021 | {
|
| 1022 | struct cma_memrange *cmr;
|
| 1023 | unsigned long ret = 0;
|
| 1024 | unsigned long i, pfn;
|
| 1025 |
|
| 1026 | cmr = find_cma_memrange(cma, pages, count);
|
| 1027 | if (!cmr)
|
| 1028 | return false;
|
| 1029 |
|
| 1030 | pfn = page_to_pfn(pages);
|
| 1031 | for (i = 0; i < count; i++, pfn++)
|
| 1032 | ret += !put_page_testzero(pfn_to_page(pfn));
|
| 1033 |
|
| 1034 | WARN(ret, "%lu pages are still in use!\n", ret);
|
| 1035 |
|
| 1036 | __cma_release_frozen(cma, cmr, pages, count);
|
| 1037 |
|
| 1038 | return true;
|
| 1039 | }
|
| 1040 | EXPORT_SYMBOL_GPL(cma_release);
|
| 1041 |
|
| 1042 | bool cma_release_frozen(struct cma *cma, const struct page *pages,
|
| 1043 | unsigned long count)
|
| 1044 | {
|
| 1045 | struct cma_memrange *cmr;
|
| 1046 |
|
| 1047 | cmr = find_cma_memrange(cma, pages, count);
|
| 1048 | if (!cmr)
|
| 1049 | return false;
|
| 1050 |
|
| 1051 | __cma_release_frozen(cma, cmr, pages, count);
|
| 1052 |
|
| 1053 | return true;
|
| 1054 | }
|
| 1055 |
|
| 1056 | int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
|
| 1057 | {
|
| 1058 | int i;
|
| 1059 |
|
| 1060 | for (i = 0; i < cma_area_count; i++) {
|
| 1061 | int ret = it(&cma_areas[i], data);
|
| 1062 |
|
| 1063 | if (ret)
|
| 1064 | return ret;
|
| 1065 | }
|
| 1066 |
|
| 1067 | return 0;
|
| 1068 | }
|
| 1069 |
|
| 1070 | bool cma_intersects(struct cma *cma, unsigned long start, unsigned long end)
|
| 1071 | {
|
| 1072 | int r;
|
| 1073 | struct cma_memrange *cmr;
|
| 1074 | unsigned long rstart, rend;
|
| 1075 |
|
| 1076 | for (r = 0; r < cma->nranges; r++) {
|
| 1077 | cmr = &cma->ranges[r];
|
| 1078 |
|
| 1079 | rstart = PFN_PHYS(cmr->base_pfn);
|
| 1080 | rend = PFN_PHYS(cmr->base_pfn + cmr->count);
|
| 1081 | if (end < rstart)
|
| 1082 | continue;
|
| 1083 | if (start >= rend)
|
| 1084 | continue;
|
| 1085 | return true;
|
| 1086 | }
|
| 1087 |
|
| 1088 | return false;
|
| 1089 | }
|
| 1090 |
|
| 1091 | |
| 1092 | * Very basic function to reserve memory from a CMA area that has not
|
| 1093 | * yet been activated. This is expected to be called early, when the
|
| 1094 | * system is single-threaded, so there is no locking. The alignment
|
| 1095 | * checking is restrictive - only pageblock-aligned areas
|
| 1096 | * (CMA_MIN_ALIGNMENT_BYTES) may be reserved through this function.
|
| 1097 | * This keeps things simple, and is enough for the current use case.
|
| 1098 | *
|
| 1099 | * The CMA bitmaps have not yet been allocated, so just start
|
| 1100 | * reserving from the bottom up, using a PFN to keep track
|
| 1101 | * of what has been reserved. Unreserving is not possible.
|
| 1102 | *
|
| 1103 | * The caller is responsible for initializing the page structures
|
| 1104 | * in the area properly, since this just points to memblock-allocated
|
| 1105 | * memory. The caller should subsequently use init_cma_pageblock to
|
| 1106 | * set the migrate type and CMA stats the pageblocks that were reserved.
|
| 1107 | *
|
| 1108 | * If the CMA area fails to activate later, memory obtained through
|
| 1109 | * this interface is not handed to the page allocator, this is
|
| 1110 | * the responsibility of the caller (e.g. like normal memblock-allocated
|
| 1111 | * memory).
|
| 1112 |
|
| 1113 | void __init *cma_reserve_early(struct cma *cma, unsigned long size)
|
| 1114 | {
|
| 1115 | int r;
|
| 1116 | struct cma_memrange *cmr;
|
| 1117 | unsigned long available;
|
| 1118 | void *ret = NULL;
|
| 1119 |
|
| 1120 | if (!cma || !cma->count)
|
| 1121 | return NULL;
|
| 1122 | |
| 1123 | * Can only be called early in init.
|
| 1124 |
|
| 1125 | if (test_bit(CMA_ACTIVATED, &cma->flags))
|
| 1126 | return NULL;
|
| 1127 |
|
| 1128 | if (!IS_ALIGNED(size, CMA_MIN_ALIGNMENT_BYTES))
|
| 1129 | return NULL;
|
| 1130 |
|
| 1131 | if (!IS_ALIGNED(size, (PAGE_SIZE << cma->order_per_bit)))
|
| 1132 | return NULL;
|
| 1133 |
|
| 1134 | size >>= PAGE_SHIFT;
|
| 1135 |
|
| 1136 | if (size > cma->available_count)
|
| 1137 | return NULL;
|
| 1138 |
|
| 1139 | for (r = 0; r < cma->nranges; r++) {
|
| 1140 | cmr = &cma->ranges[r];
|
| 1141 | available = cmr->count - (cmr->early_pfn - cmr->base_pfn);
|
| 1142 | if (size <= available) {
|
| 1143 | ret = phys_to_virt(PFN_PHYS(cmr->early_pfn));
|
| 1144 | cmr->early_pfn += size;
|
| 1145 | cma->available_count -= size;
|
| 1146 | return ret;
|
| 1147 | }
|
| 1148 | }
|
| 1149 |
|
| 1150 | return ret;
|
| 1151 | }
|