| 1 | |
| 2 | |
| 3 | * mm/percpu-vm.c - vmalloc area based chunk allocation
|
| 4 | *
|
| 5 | * Copyright (C) 2010 SUSE Linux Products GmbH
|
| 6 | * Copyright (C) 2010 Tejun Heo <[email protected]>
|
| 7 | *
|
| 8 | * Chunks are mapped into vmalloc areas and populated page by page.
|
| 9 | * This is the default chunk allocator.
|
| 10 |
|
| 11 | #include "internal.h"
|
| 12 | #include "vmalloc.h"
|
| 13 |
|
| 14 | static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
|
| 15 | unsigned int cpu, int page_idx)
|
| 16 | {
|
| 17 |
|
| 18 | WARN_ON(chunk->immutable);
|
| 19 |
|
| 20 | return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
|
| 21 | }
|
| 22 |
|
| 23 | |
| 24 | * pcpu_get_pages - get temp pages array
|
| 25 | * @gfp: allocation flags passed to the underlying allocator, 0 to only
|
| 26 | * return the cached array
|
| 27 | *
|
| 28 | * Returns pointer to array of pointers to struct page which can be indexed
|
| 29 | * with pcpu_page_idx(). Note that there is only one array and accesses
|
| 30 | * should be serialized by pcpu_alloc_mutex.
|
| 31 | *
|
| 32 | * RETURNS:
|
| 33 | * Pointer to temp pages array on success.
|
| 34 |
|
| 35 | static struct page **pcpu_get_pages(gfp_t gfp)
|
| 36 | {
|
| 37 | static struct page **pages;
|
| 38 | size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
|
| 39 |
|
| 40 | lockdep_assert_held(&pcpu_alloc_mutex);
|
| 41 |
|
| 42 | if (!pages && gfp)
|
| 43 | pages = pcpu_mem_zalloc(pages_size, gfp);
|
| 44 | return pages;
|
| 45 | }
|
| 46 |
|
| 47 | static struct page **pcpu_get_pages_cached(void)
|
| 48 | {
|
| 49 | return pcpu_get_pages(0);
|
| 50 | }
|
| 51 |
|
| 52 | |
| 53 | * pcpu_free_pages - free pages which were allocated for @chunk
|
| 54 | * @chunk: chunk pages were allocated for
|
| 55 | * @pages: array of pages to be freed, indexed by pcpu_page_idx()
|
| 56 | * @page_start: page index of the first page to be freed
|
| 57 | * @page_end: page index of the last page to be freed + 1
|
| 58 | *
|
| 59 | * Free pages [@page_start and @page_end) in @pages for all units.
|
| 60 | * The pages were allocated for @chunk.
|
| 61 |
|
| 62 | static void pcpu_free_pages(struct pcpu_chunk *chunk,
|
| 63 | struct page **pages, int page_start, int page_end)
|
| 64 | {
|
| 65 | unsigned int cpu;
|
| 66 | int i;
|
| 67 |
|
| 68 | for_each_possible_cpu(cpu) {
|
| 69 | for (i = page_start; i < page_end; i++) {
|
| 70 | struct page *page = pages[pcpu_page_idx(cpu, i)];
|
| 71 |
|
| 72 | if (page)
|
| 73 | __free_page(page);
|
| 74 | }
|
| 75 | }
|
| 76 | }
|
| 77 |
|
| 78 | |
| 79 | * pcpu_alloc_pages - allocates pages for @chunk
|
| 80 | * @chunk: target chunk
|
| 81 | * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
|
| 82 | * @page_start: page index of the first page to be allocated
|
| 83 | * @page_end: page index of the last page to be allocated + 1
|
| 84 | * @gfp: allocation flags passed to the underlying allocator
|
| 85 | *
|
| 86 | * Allocate pages [@page_start,@page_end) into @pages for all units.
|
| 87 | * The allocation is for @chunk. Percpu core doesn't care about the
|
| 88 | * content of @pages and will pass it verbatim to pcpu_map_pages().
|
| 89 |
|
| 90 | static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
|
| 91 | struct page **pages, int page_start, int page_end,
|
| 92 | gfp_t gfp)
|
| 93 | {
|
| 94 | unsigned int cpu, tcpu;
|
| 95 | int i;
|
| 96 |
|
| 97 | gfp |= __GFP_HIGHMEM;
|
| 98 |
|
| 99 | for_each_possible_cpu(cpu) {
|
| 100 | for (i = page_start; i < page_end; i++) {
|
| 101 | struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
|
| 102 |
|
| 103 | *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
|
| 104 | if (!*pagep)
|
| 105 | goto err;
|
| 106 | }
|
| 107 | }
|
| 108 | return 0;
|
| 109 |
|
| 110 | err:
|
| 111 | while (--i >= page_start)
|
| 112 | __free_page(pages[pcpu_page_idx(cpu, i)]);
|
| 113 |
|
| 114 | for_each_possible_cpu(tcpu) {
|
| 115 | if (tcpu == cpu)
|
| 116 | break;
|
| 117 | for (i = page_start; i < page_end; i++)
|
| 118 | __free_page(pages[pcpu_page_idx(tcpu, i)]);
|
| 119 | }
|
| 120 | return -ENOMEM;
|
| 121 | }
|
| 122 |
|
| 123 | |
| 124 | * pcpu_pre_unmap_flush - flush cache prior to unmapping
|
| 125 | * @chunk: chunk the regions to be flushed belongs to
|
| 126 | * @page_start: page index of the first page to be flushed
|
| 127 | * @page_end: page index of the last page to be flushed + 1
|
| 128 | *
|
| 129 | * Pages in [@page_start,@page_end) of @chunk are about to be
|
| 130 | * unmapped. Flush cache. As each flushing trial can be very
|
| 131 | * expensive, issue flush on the whole region at once rather than
|
| 132 | * doing it for each cpu. This could be an overkill but is more
|
| 133 | * scalable.
|
| 134 |
|
| 135 | static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
|
| 136 | int page_start, int page_end)
|
| 137 | {
|
| 138 | flush_cache_vunmap(
|
| 139 | pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
|
| 140 | pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
|
| 141 | }
|
| 142 |
|
| 143 | static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
|
| 144 | {
|
| 145 | vunmap_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT));
|
| 146 | }
|
| 147 |
|
| 148 | |
| 149 | * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
|
| 150 | * @chunk: chunk of interest
|
| 151 | * @pages: pages array which can be used to pass information to free
|
| 152 | * @page_start: page index of the first page to unmap
|
| 153 | * @page_end: page index of the last page to unmap + 1
|
| 154 | *
|
| 155 | * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
|
| 156 | * Corresponding elements in @pages were cleared by the caller and can
|
| 157 | * be used to carry information to pcpu_free_pages() which will be
|
| 158 | * called after all unmaps are finished. The caller should call
|
| 159 | * proper pre/post flush functions.
|
| 160 |
|
| 161 | static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
|
| 162 | struct page **pages, int page_start, int page_end)
|
| 163 | {
|
| 164 | unsigned int cpu;
|
| 165 | int i;
|
| 166 |
|
| 167 | for_each_possible_cpu(cpu) {
|
| 168 | for (i = page_start; i < page_end; i++) {
|
| 169 | struct page *page;
|
| 170 |
|
| 171 | page = pcpu_chunk_page(chunk, cpu, i);
|
| 172 | WARN_ON(!page);
|
| 173 | pages[pcpu_page_idx(cpu, i)] = page;
|
| 174 | }
|
| 175 | __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
|
| 176 | page_end - page_start);
|
| 177 | }
|
| 178 | }
|
| 179 |
|
| 180 | |
| 181 | * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
|
| 182 | * @chunk: pcpu_chunk the regions to be flushed belong to
|
| 183 | * @page_start: page index of the first page to be flushed
|
| 184 | * @page_end: page index of the last page to be flushed + 1
|
| 185 | *
|
| 186 | * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
|
| 187 | * TLB for the regions. This can be skipped if the area is to be
|
| 188 | * returned to vmalloc as vmalloc will handle TLB flushing lazily.
|
| 189 | *
|
| 190 | * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
|
| 191 | * for the whole region.
|
| 192 |
|
| 193 | static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
|
| 194 | int page_start, int page_end)
|
| 195 | {
|
| 196 | flush_tlb_kernel_range(
|
| 197 | pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
|
| 198 | pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
|
| 199 | }
|
| 200 |
|
| 201 | static int __pcpu_map_pages(unsigned long addr, struct page **pages,
|
| 202 | int nr_pages, gfp_t gfp)
|
| 203 | {
|
| 204 | unsigned int flags;
|
| 205 | int ret;
|
| 206 |
|
| 207 | |
| 208 | * The vmalloc page table allocation path does not pass @gfp down
|
| 209 | * explicitly. Apply the corresponding memalloc scope so implicit
|
| 210 | * page table allocations preserve NOFS/NOIO constraints.
|
| 211 |
|
| 212 | flags = memalloc_apply_gfp_scope(gfp);
|
| 213 | ret = vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT),
|
| 214 | PAGE_KERNEL, pages, PAGE_SHIFT, gfp);
|
| 215 | memalloc_restore_scope(flags);
|
| 216 |
|
| 217 | return ret;
|
| 218 | }
|
| 219 |
|
| 220 | |
| 221 | * pcpu_map_pages - map pages into a pcpu_chunk
|
| 222 | * @chunk: chunk of interest
|
| 223 | * @pages: pages array containing pages to be mapped
|
| 224 | * @page_start: page index of the first page to map
|
| 225 | * @page_end: page index of the last page to map + 1
|
| 226 | * @gfp: allocation flags passed to the underlying allocator
|
| 227 | *
|
| 228 | * For each cpu, map pages [@page_start,@page_end) into @chunk. The
|
| 229 | * caller is responsible for calling pcpu_post_map_flush() after all
|
| 230 | * mappings are complete.
|
| 231 | *
|
| 232 | * This function is responsible for setting up whatever is necessary for
|
| 233 | * reverse lookup (addr -> chunk).
|
| 234 |
|
| 235 | static int pcpu_map_pages(struct pcpu_chunk *chunk, struct page **pages,
|
| 236 | int page_start, int page_end, gfp_t gfp)
|
| 237 | {
|
| 238 | unsigned int cpu, tcpu;
|
| 239 | int i, err;
|
| 240 |
|
| 241 | for_each_possible_cpu(cpu) {
|
| 242 | err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
|
| 243 | &pages[pcpu_page_idx(cpu, page_start)],
|
| 244 | page_end - page_start, gfp);
|
| 245 | if (err < 0)
|
| 246 | goto err;
|
| 247 |
|
| 248 | for (i = page_start; i < page_end; i++)
|
| 249 | pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
|
| 250 | chunk);
|
| 251 | }
|
| 252 | return 0;
|
| 253 | err:
|
| 254 | for_each_possible_cpu(tcpu) {
|
| 255 | __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
|
| 256 | page_end - page_start);
|
| 257 | if (tcpu == cpu)
|
| 258 | break;
|
| 259 | }
|
| 260 | pcpu_post_unmap_tlb_flush(chunk, page_start, page_end);
|
| 261 | return err;
|
| 262 | }
|
| 263 |
|
| 264 | |
| 265 | * pcpu_post_map_flush - flush cache after mapping
|
| 266 | * @chunk: pcpu_chunk the regions to be flushed belong to
|
| 267 | * @page_start: page index of the first page to be flushed
|
| 268 | * @page_end: page index of the last page to be flushed + 1
|
| 269 | *
|
| 270 | * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
|
| 271 | * cache.
|
| 272 | *
|
| 273 | * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
|
| 274 | * for the whole region.
|
| 275 |
|
| 276 | static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
|
| 277 | int page_start, int page_end)
|
| 278 | {
|
| 279 | flush_cache_vmap(
|
| 280 | pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
|
| 281 | pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
|
| 282 | }
|
| 283 |
|
| 284 | |
| 285 | * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
|
| 286 | * @chunk: chunk of interest
|
| 287 | * @page_start: the start page
|
| 288 | * @page_end: the end page
|
| 289 | * @gfp: allocation flags passed to the underlying memory allocator
|
| 290 | *
|
| 291 | * For each cpu, populate and map pages [@page_start,@page_end) into
|
| 292 | * @chunk.
|
| 293 | *
|
| 294 | * CONTEXT:
|
| 295 | * pcpu_alloc_mutex, does @gfp allocation.
|
| 296 |
|
| 297 | static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
|
| 298 | int page_start, int page_end, gfp_t gfp)
|
| 299 | {
|
| 300 | struct page **pages;
|
| 301 |
|
| 302 | pages = pcpu_get_pages(gfp);
|
| 303 | if (!pages)
|
| 304 | return -ENOMEM;
|
| 305 |
|
| 306 | if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))
|
| 307 | return -ENOMEM;
|
| 308 |
|
| 309 | if (pcpu_map_pages(chunk, pages, page_start, page_end, gfp)) {
|
| 310 | pcpu_free_pages(chunk, pages, page_start, page_end);
|
| 311 | return -ENOMEM;
|
| 312 | }
|
| 313 | pcpu_post_map_flush(chunk, page_start, page_end);
|
| 314 |
|
| 315 | return 0;
|
| 316 | }
|
| 317 |
|
| 318 | |
| 319 | * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
|
| 320 | * @chunk: chunk to depopulate
|
| 321 | * @page_start: the start page
|
| 322 | * @page_end: the end page
|
| 323 | *
|
| 324 | * For each cpu, depopulate and unmap pages [@page_start,@page_end)
|
| 325 | * from @chunk.
|
| 326 | *
|
| 327 | * Caller is required to call pcpu_post_unmap_tlb_flush() if not returning the
|
| 328 | * region back to vmalloc() which will lazily flush the tlb.
|
| 329 | *
|
| 330 | * CONTEXT:
|
| 331 | * pcpu_alloc_mutex.
|
| 332 |
|
| 333 | static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
|
| 334 | int page_start, int page_end)
|
| 335 | {
|
| 336 | struct page **pages;
|
| 337 |
|
| 338 | |
| 339 | * If control reaches here, there must have been at least one
|
| 340 | * successful population attempt so the temp pages array must
|
| 341 | * be available now.
|
| 342 |
|
| 343 | pages = pcpu_get_pages_cached();
|
| 344 | BUG_ON(!pages);
|
| 345 |
|
| 346 |
|
| 347 | pcpu_pre_unmap_flush(chunk, page_start, page_end);
|
| 348 |
|
| 349 | pcpu_unmap_pages(chunk, pages, page_start, page_end);
|
| 350 |
|
| 351 | pcpu_free_pages(chunk, pages, page_start, page_end);
|
| 352 | }
|
| 353 |
|
| 354 | static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
|
| 355 | {
|
| 356 | struct pcpu_chunk *chunk;
|
| 357 | struct vm_struct **vms;
|
| 358 |
|
| 359 | chunk = pcpu_alloc_chunk(gfp);
|
| 360 | if (!chunk)
|
| 361 | return NULL;
|
| 362 |
|
| 363 | vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
|
| 364 | pcpu_nr_groups, pcpu_atom_size, gfp);
|
| 365 | if (!vms) {
|
| 366 | pcpu_free_chunk(chunk);
|
| 367 | return NULL;
|
| 368 | }
|
| 369 |
|
| 370 | chunk->data = vms;
|
| 371 | chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
|
| 372 |
|
| 373 | pcpu_stats_chunk_alloc();
|
| 374 | trace_percpu_create_chunk(chunk->base_addr);
|
| 375 |
|
| 376 | return chunk;
|
| 377 | }
|
| 378 |
|
| 379 | static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
|
| 380 | {
|
| 381 | if (!chunk)
|
| 382 | return;
|
| 383 |
|
| 384 | pcpu_stats_chunk_dealloc();
|
| 385 | trace_percpu_destroy_chunk(chunk->base_addr);
|
| 386 |
|
| 387 | if (chunk->data)
|
| 388 | pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
|
| 389 | pcpu_free_chunk(chunk);
|
| 390 | }
|
| 391 |
|
| 392 | static struct page *pcpu_addr_to_page(void *addr)
|
| 393 | {
|
| 394 | return vmalloc_to_page(addr);
|
| 395 | }
|
| 396 |
|
| 397 | static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
|
| 398 | {
|
| 399 |
|
| 400 | return 0;
|
| 401 | }
|
| 402 |
|
| 403 | |
| 404 | * pcpu_should_reclaim_chunk - determine if a chunk should go into reclaim
|
| 405 | * @chunk: chunk of interest
|
| 406 | *
|
| 407 | * This is the entry point for percpu reclaim. If a chunk qualifies, it is then
|
| 408 | * isolated and managed in separate lists at the back of pcpu_slot: sidelined
|
| 409 | * and to_depopulate respectively. The to_depopulate list holds chunks slated
|
| 410 | * for depopulation. They no longer contribute to pcpu_nr_empty_pop_pages once
|
| 411 | * they are on this list. Once depopulated, they are moved onto the sidelined
|
| 412 | * list which enables them to be pulled back in for allocation if no other chunk
|
| 413 | * can suffice the allocation.
|
| 414 |
|
| 415 | static bool pcpu_should_reclaim_chunk(struct pcpu_chunk *chunk)
|
| 416 | {
|
| 417 |
|
| 418 | if (chunk == pcpu_first_chunk || chunk == pcpu_reserved_chunk)
|
| 419 | return false;
|
| 420 |
|
| 421 | |
| 422 | * If it is isolated, it may be on the sidelined list so move it back to
|
| 423 | * the to_depopulate list. If we hit at least 1/4 pages empty pages AND
|
| 424 | * there is no system-wide shortage of empty pages aside from this
|
| 425 | * chunk, move it to the to_depopulate list.
|
| 426 |
|
| 427 | return ((chunk->isolated && chunk->nr_empty_pop_pages) ||
|
| 428 | (pcpu_nr_empty_pop_pages >
|
| 429 | (PCPU_EMPTY_POP_PAGES_HIGH + chunk->nr_empty_pop_pages) &&
|
| 430 | chunk->nr_empty_pop_pages >= chunk->nr_pages / 4));
|
| 431 | }
|