projects/linux·blob 2c455ddc · refs

772 lines · 22.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * memory buffer pool support. Such pools are mostly used
4 * for guaranteed, deadlock-free memory allocations during
5 * extreme VM load.
6 *
7 * started by Ingo Molnar, Copyright (C) 2001
8 * debugging by David Rientjes, Copyright (C) 2015
9 */
10#include <linux/fault-inject.h>
11#include <linux/mm.h>
12#include <linux/slab.h>
13#include <linux/highmem.h>
14#include <linux/kasan.h>
15#include <linux/kmemleak.h>
16#include <linux/export.h>
17#include <linux/mempool.h>
18#include <linux/writeback.h>
19#include <linux/static_key.h>
20#include <linux/init.h>
21#include "slab.h"
22
23static DECLARE_FAULT_ATTR(fail_mempool_alloc);
24static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk);
25
26/*
27 * Debugging support for mempool using static key.
28 *
29 * This allows enabling mempool debug at boot time via:
30 * mempool_debug
31 */
32static DEFINE_STATIC_KEY_FALSE(mempool_debug_enabled);
33
34static int __init mempool_debug_setup(char *str)
35{
36 static_branch_enable(&mempool_debug_enabled);
37 return 1;
38}
39__setup("mempool_debug", mempool_debug_setup);
40
41static int __init mempool_faul_inject_init(void)
42{
43 int error;
44
45 error = PTR_ERR_OR_ZERO(fault_create_debugfs_attr("fail_mempool_alloc",
46 NULL, &fail_mempool_alloc));
47 if (error)
48 return error;
49
50 /* booting will fail on error return here, don't bother to cleanup */
51 return PTR_ERR_OR_ZERO(
52 fault_create_debugfs_attr("fail_mempool_alloc_bulk", NULL,
53 &fail_mempool_alloc_bulk));
54}
55late_initcall(mempool_faul_inject_init);
56
57static void poison_error(struct mempool *pool, void *element, size_t size,
58 size_t byte)
59{
60 const int nr = pool->curr_nr;
61 const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0);
62 const int end = min_t(int, byte + (BITS_PER_LONG / 8), size);
63 int i;
64
65 pr_err("BUG: mempool element poison mismatch\n");
66 pr_err("Mempool %p size %zu\n", pool, size);
67 pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
68 for (i = start; i < end; i++)
69 pr_cont("%x ", *(u8 *)(element + i));
70 pr_cont("%s\n", end < size ? "..." : "");
71 dump_stack();
72}
73
74static void __check_element(struct mempool *pool, void *element, size_t size)
75{
76 u8 *obj = element;
77 size_t i;
78
79 for (i = 0; i < size; i++) {
80 u8 exp = (i < size - 1) ? POISON_FREE : POISON_END;
81
82 if (obj[i] != exp) {
83 poison_error(pool, element, size, i);
84 return;
85 }
86 }
87 memset(obj, POISON_INUSE, size);
88}
89
90static void check_element(struct mempool *pool, void *element)
91{
92 /* Skip checking: KASAN might save its metadata in the element. */
93 if (kasan_enabled())
94 return;
95
96 /* Mempools backed by slab allocator */
97 if (pool->free == mempool_kfree) {
98 __check_element(pool, element, (size_t)pool->pool_data);
99 } else if (pool->free == mempool_free_slab) {
100 __check_element(pool, element, kmem_cache_size(pool->pool_data));
101 } else if (pool->free == mempool_free_pages) {
102 /* Mempools backed by page allocator */
103 int order = (int)(long)pool->pool_data;
104
105#ifdef CONFIG_HIGHMEM
106 for (int i = 0; i < (1 << order); i++) {
107 struct page *page = (struct page *)element;
108 void *addr = kmap_local_page(page + i);
109
110 __check_element(pool, addr, PAGE_SIZE);
111 kunmap_local(addr);
112 }
113#else
114 void *addr = page_address((struct page *)element);
115
116 __check_element(pool, addr, PAGE_SIZE << order);
117#endif
118 }
119}
120
121static void __poison_element(void *element, size_t size)
122{
123 u8 *obj = element;
124
125 memset(obj, POISON_FREE, size - 1);
126 obj[size - 1] = POISON_END;
127}
128
129static void poison_element(struct mempool *pool, void *element)
130{
131 /* Skip poisoning: KASAN might save its metadata in the element. */
132 if (kasan_enabled())
133 return;
134
135 /* Mempools backed by slab allocator */
136 if (pool->alloc == mempool_kmalloc) {
137 __poison_element(element, (size_t)pool->pool_data);
138 } else if (pool->alloc == mempool_alloc_slab) {
139 __poison_element(element, kmem_cache_size(pool->pool_data));
140 } else if (pool->alloc == mempool_alloc_pages) {
141 /* Mempools backed by page allocator */
142 int order = (int)(long)pool->pool_data;
143
144#ifdef CONFIG_HIGHMEM
145 for (int i = 0; i < (1 << order); i++) {
146 struct page *page = (struct page *)element;
147 void *addr = kmap_local_page(page + i);
148
149 __poison_element(addr, PAGE_SIZE);
150 kunmap_local(addr);
151 }
152#else
153 void *addr = page_address((struct page *)element);
154
155 __poison_element(addr, PAGE_SIZE << order);
156#endif
157 }
158}
159
160static __always_inline bool kasan_poison_element(struct mempool *pool,
161 void *element)
162{
163 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
164 return kasan_mempool_poison_object(element);
165 else if (pool->alloc == mempool_alloc_pages)
166 return kasan_mempool_poison_pages(element,
167 (unsigned long)pool->pool_data);
168 return true;
169}
170
171static void kasan_unpoison_element(struct mempool *pool, void *element)
172{
173 if (pool->alloc == mempool_kmalloc)
174 kasan_mempool_unpoison_object(element, (size_t)pool->pool_data);
175 else if (pool->alloc == mempool_alloc_slab)
176 kasan_mempool_unpoison_object(element,
177 kmem_cache_size(pool->pool_data));
178 else if (pool->alloc == mempool_alloc_pages)
179 kasan_mempool_unpoison_pages(element,
180 (unsigned long)pool->pool_data);
181}
182
183static __always_inline void add_element(struct mempool *pool, void *element)
184{
185 BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr);
186
187 if (static_branch_unlikely(&mempool_debug_enabled))
188 poison_element(pool, element);
189
190 if (kasan_poison_element(pool, element))
191 pool->elements[pool->curr_nr++] = element;
192}
193
194static void *remove_element(struct mempool *pool)
195{
196 void *element = pool->elements[--pool->curr_nr];
197
198 BUG_ON(pool->curr_nr < 0);
199 kasan_unpoison_element(pool, element);
200
201 if (static_branch_unlikely(&mempool_debug_enabled))
202 check_element(pool, element);
203 return element;
204}
205
206/**
207 * mempool_exit - exit a mempool initialized with mempool_init()
208 * @pool: pointer to the memory pool which was initialized with
209 * mempool_init().
210 *
211 * Free all reserved elements in @pool and @pool itself. This function
212 * only sleeps if the free_fn() function sleeps.
213 *
214 * May be called on a zeroed but uninitialized mempool (i.e. allocated with
215 * kzalloc()).
216 */
217void mempool_exit(struct mempool *pool)
218{
219 while (pool->curr_nr) {
220 void *element = remove_element(pool);
221 pool->free(element, pool->pool_data);
222 }
223 kfree(pool->elements);
224 pool->elements = NULL;
225}
226EXPORT_SYMBOL(mempool_exit);
227
228/**
229 * mempool_destroy - deallocate a memory pool
230 * @pool: pointer to the memory pool which was allocated via
231 * mempool_create().
232 *
233 * Free all reserved elements in @pool and @pool itself. This function
234 * only sleeps if the free_fn() function sleeps.
235 */
236void mempool_destroy(struct mempool *pool)
237{
238 if (unlikely(!pool))
239 return;
240
241 mempool_exit(pool);
242 kfree(pool);
243}
244EXPORT_SYMBOL(mempool_destroy);
245
246int mempool_init_node(struct mempool *pool, int min_nr,
247 mempool_alloc_t *alloc_fn, mempool_free_t *free_fn,
248 void *pool_data, gfp_t gfp_mask, int node_id)
249{
250 spin_lock_init(&pool->lock);
251 pool->min_nr = min_nr;
252 pool->pool_data = pool_data;
253 pool->alloc = alloc_fn;
254 pool->free = free_fn;
255 init_waitqueue_head(&pool->wait);
256 /*
257 * max() used here to ensure storage for at least 1 element to support
258 * zero minimum pool
259 */
260 pool->elements = kmalloc_array_node(max(1, min_nr), sizeof(void *),
261 gfp_mask, node_id);
262 if (!pool->elements)
263 return -ENOMEM;
264
265 /*
266 * First pre-allocate the guaranteed number of buffers,
267 * also pre-allocate 1 element for zero minimum pool.
268 */
269 while (pool->curr_nr < max(1, pool->min_nr)) {
270 void *element;
271
272 element = pool->alloc(gfp_mask, pool->pool_data);
273 if (unlikely(!element)) {
274 mempool_exit(pool);
275 return -ENOMEM;
276 }
277 add_element(pool, element);
278 }
279
280 return 0;
281}
282EXPORT_SYMBOL(mempool_init_node);
283
284/**
285 * mempool_init - initialize a memory pool
286 * @pool: pointer to the memory pool that should be initialized
287 * @min_nr: the minimum number of elements guaranteed to be
288 * allocated for this pool.
289 * @alloc_fn: user-defined element-allocation function.
290 * @free_fn: user-defined element-freeing function.
291 * @pool_data: optional private data available to the user-defined functions.
292 *
293 * Like mempool_create(), but initializes the pool in (i.e. embedded in another
294 * structure).
295 *
296 * Return: %0 on success, negative error code otherwise.
297 */
298int mempool_init_noprof(struct mempool *pool, int min_nr,
299 mempool_alloc_t *alloc_fn, mempool_free_t *free_fn,
300 void *pool_data)
301{
302 return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
303 pool_data, GFP_KERNEL, NUMA_NO_NODE);
304
305}
306EXPORT_SYMBOL(mempool_init_noprof);
307
308/**
309 * mempool_create_node - create a memory pool
310 * @min_nr: the minimum number of elements guaranteed to be
311 * allocated for this pool.
312 * @alloc_fn: user-defined element-allocation function.
313 * @free_fn: user-defined element-freeing function.
314 * @pool_data: optional private data available to the user-defined functions.
315 * @gfp_mask: memory allocation flags
316 * @node_id: numa node to allocate on
317 *
318 * this function creates and allocates a guaranteed size, preallocated
319 * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
320 * functions. This function might sleep. Both the alloc_fn() and the free_fn()
321 * functions might sleep - as long as the mempool_alloc() function is not called
322 * from IRQ contexts.
323 *
324 * Return: pointer to the created memory pool object or %NULL on error.
325 */
326struct mempool *mempool_create_node_noprof(int min_nr,
327 mempool_alloc_t *alloc_fn, mempool_free_t *free_fn,
328 void *pool_data, gfp_t gfp_mask, int node_id)
329{
330 struct mempool *pool;
331
332 pool = kmalloc_node_noprof(sizeof(*pool), gfp_mask | __GFP_ZERO, node_id);
333 if (!pool)
334 return NULL;
335
336 if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
337 gfp_mask, node_id)) {
338 kfree(pool);
339 return NULL;
340 }
341
342 return pool;
343}
344EXPORT_SYMBOL(mempool_create_node_noprof);
345
346/**
347 * mempool_resize - resize an existing memory pool
348 * @pool: pointer to the memory pool which was allocated via
349 * mempool_create().
350 * @new_min_nr: the new minimum number of elements guaranteed to be
351 * allocated for this pool.
352 *
353 * This function shrinks/grows the pool. In the case of growing,
354 * it cannot be guaranteed that the pool will be grown to the new
355 * size immediately, but new mempool_free() calls will refill it.
356 * This function may sleep.
357 *
358 * Note, the caller must guarantee that no mempool_destroy is called
359 * while this function is running. mempool_alloc() & mempool_free()
360 * might be called (eg. from IRQ contexts) while this function executes.
361 *
362 * Return: %0 on success, negative error code otherwise.
363 */
364int mempool_resize(struct mempool *pool, int new_min_nr)
365{
366 void *element;
367 void **new_elements;
368 unsigned long flags;
369
370 BUG_ON(new_min_nr <= 0);
371 might_sleep();
372
373 spin_lock_irqsave(&pool->lock, flags);
374 if (new_min_nr <= pool->min_nr) {
375 while (new_min_nr < pool->curr_nr) {
376 element = remove_element(pool);
377 spin_unlock_irqrestore(&pool->lock, flags);
378 pool->free(element, pool->pool_data);
379 spin_lock_irqsave(&pool->lock, flags);
380 }
381 pool->min_nr = new_min_nr;
382 goto out_unlock;
383 }
384 spin_unlock_irqrestore(&pool->lock, flags);
385
386 /* Grow the pool */
387 new_elements = kmalloc_objs(*new_elements, new_min_nr);
388 if (!new_elements)
389 return -ENOMEM;
390
391 spin_lock_irqsave(&pool->lock, flags);
392 if (unlikely(new_min_nr <= pool->min_nr)) {
393 /* Raced, other resize will do our work */
394 spin_unlock_irqrestore(&pool->lock, flags);
395 kfree(new_elements);
396 goto out;
397 }
398 memcpy(new_elements, pool->elements,
399 pool->curr_nr * sizeof(*new_elements));
400 kfree(pool->elements);
401 pool->elements = new_elements;
402 pool->min_nr = new_min_nr;
403
404 while (pool->curr_nr < pool->min_nr) {
405 spin_unlock_irqrestore(&pool->lock, flags);
406 element = pool->alloc(GFP_KERNEL, pool->pool_data);
407 if (!element)
408 goto out;
409 spin_lock_irqsave(&pool->lock, flags);
410 if (pool->curr_nr < pool->min_nr) {
411 add_element(pool, element);
412 } else {
413 spin_unlock_irqrestore(&pool->lock, flags);
414 pool->free(element, pool->pool_data); /* Raced */
415 goto out;
416 }
417 }
418out_unlock:
419 spin_unlock_irqrestore(&pool->lock, flags);
420out:
421 return 0;
422}
423EXPORT_SYMBOL(mempool_resize);
424
425static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems,
426 unsigned int count, unsigned int allocated,
427 gfp_t gfp_mask)
428{
429 unsigned long flags;
430 unsigned int i;
431
432 spin_lock_irqsave(&pool->lock, flags);
433 if (unlikely(pool->curr_nr < count - allocated))
434 goto fail;
435 while (allocated < count)
436 elems[allocated++] = remove_element(pool);
437 spin_unlock_irqrestore(&pool->lock, flags);
438
439 /* Paired with rmb in mempool_free(), read comment there. */
440 smp_wmb();
441
442 /*
443 * Update the allocation stack trace as this is more useful for
444 * debugging.
445 */
446 for (i = 0; i < count; i++)
447 kmemleak_update_trace(elems[i]);
448 return allocated;
449
450fail:
451 if (gfp_mask & __GFP_DIRECT_RECLAIM) {
452 DEFINE_WAIT(wait);
453
454 prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
455 spin_unlock_irqrestore(&pool->lock, flags);
456
457 /*
458 * Wait for someone else to return an element to @pool, but wake
459 * up occasionally as memory pressure might have reduced even
460 * and the normal allocation in alloc_fn could succeed even if
461 * no element was returned.
462 */
463 io_schedule_timeout(5 * HZ);
464 finish_wait(&pool->wait, &wait);
465 } else {
466 /* We must not sleep if __GFP_DIRECT_RECLAIM is not set. */
467 spin_unlock_irqrestore(&pool->lock, flags);
468 }
469
470 return allocated;
471}
472
473/*
474 * Adjust the gfp flags for mempool allocations, as we never want to dip into
475 * the global emergency reserves or retry in the page allocator.
476 *
477 * The first pass also doesn't want to go reclaim, but the next passes do, so
478 * return a separate subset for that first iteration.
479 */
480static inline gfp_t mempool_adjust_gfp(gfp_t *gfp_mask)
481{
482 *gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
483 return *gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
484}
485
486/**
487 * mempool_alloc_bulk - allocate multiple elements from a memory pool
488 * @pool: pointer to the memory pool
489 * @elems: partially or fully populated elements array
490 * @count: number of entries in @elem that need to be allocated
491 *
492 * Allocate @count elements into @elems. This is done by first calling into the
493 * alloc_fn supplied at pool initialization time, and dipping into the reserved
494 * pool when alloc_fn fails to allocate an element.
495 *
496 * On return all @count elements in @elems will be populated.
497 *
498 * Return: Always 0. If it wasn't for %$#^$ alloc tags, it would return void.
499 */
500int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems,
501 unsigned int count)
502{
503 gfp_t gfp_mask = GFP_KERNEL;
504 gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask);
505 unsigned int allocated = 0;
506
507 VM_WARN_ON_ONCE(count > pool->min_nr);
508 might_alloc(gfp_mask);
509
510 /*
511 * If an error is injected, fail all elements in a bulk allocation so
512 * that we stress the multiple elements missing path.
513 */
514 if (should_fail_ex(&fail_mempool_alloc_bulk, 1, FAULT_NOWARN)) {
515 pr_info("forcing mempool usage for %pS\n",
516 (void *)_RET_IP_);
517 goto use_pool;
518 }
519
520repeat_alloc:
521 /*
522 * Try to allocate the elements using the allocation callback first as
523 * that might succeed even when the caller's bulk allocation did not.
524 */
525 while (allocated < count) {
526 elems[allocated] = pool->alloc(gfp_temp, pool->pool_data);
527 if (unlikely(!elems[allocated]))
528 goto use_pool;
529 allocated++;
530 }
531
532 return 0;
533
534use_pool:
535 allocated = mempool_alloc_from_pool(pool, elems, count, allocated,
536 gfp_temp);
537 gfp_temp = gfp_mask;
538 goto repeat_alloc;
539}
540EXPORT_SYMBOL_GPL(mempool_alloc_bulk_noprof);
541
542/**
543 * mempool_alloc - allocate an element from a memory pool
544 * @pool: pointer to the memory pool
545 * @gfp_mask: GFP_* flags. %__GFP_ZERO is not supported.
546 *
547 * Allocate an element from @pool. This is done by first calling into the
548 * alloc_fn supplied at pool initialization time, and dipping into the reserved
549 * pool when alloc_fn fails to allocate an element.
550 *
551 * This function only sleeps if the alloc_fn callback sleeps, or when waiting
552 * for elements to become available in the pool.
553 *
554 * Return: pointer to the allocated element or %NULL when failing to allocate
555 * an element. Allocation failure can only happen when @gfp_mask does not
556 * include %__GFP_DIRECT_RECLAIM.
557 */
558void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask)
559{
560 gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask);
561 void *element;
562
563 VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
564 might_alloc(gfp_mask);
565
566repeat_alloc:
567 if (should_fail_ex(&fail_mempool_alloc, 1, FAULT_NOWARN)) {
568 pr_info("forcing mempool usage for %pS\n",
569 (void *)_RET_IP_);
570 element = NULL;
571 } else {
572 element = pool->alloc(gfp_temp, pool->pool_data);
573 }
574
575 if (unlikely(!element)) {
576 /*
577 * Try to allocate an element from the pool.
578 *
579 * The first pass won't have __GFP_DIRECT_RECLAIM and won't
580 * sleep in mempool_alloc_from_pool. Retry the allocation
581 * with all flags set in that case.
582 */
583 if (!mempool_alloc_from_pool(pool, &element, 1, 0, gfp_temp)) {
584 if (gfp_temp != gfp_mask) {
585 gfp_temp = gfp_mask;
586 goto repeat_alloc;
587 }
588 if (gfp_mask & __GFP_DIRECT_RECLAIM) {
589 goto repeat_alloc;
590 }
591 }
592 }
593
594 return element;
595}
596EXPORT_SYMBOL(mempool_alloc_noprof);
597
598/**
599 * mempool_alloc_preallocated - allocate an element from preallocated elements
600 * belonging to a memory pool
601 * @pool: pointer to the memory pool
602 *
603 * This function is similar to mempool_alloc(), but it only attempts allocating
604 * an element from the preallocated elements. It only takes a single spinlock_t
605 * and immediately returns if no preallocated elements are available.
606 *
607 * Return: pointer to the allocated element or %NULL if no elements are
608 * available.
609 */
610void *mempool_alloc_preallocated(struct mempool *pool)
611{
612 void *element = NULL;
613
614 mempool_alloc_from_pool(pool, &element, 1, 0, GFP_NOWAIT);
615 return element;
616}
617EXPORT_SYMBOL(mempool_alloc_preallocated);
618
619/**
620 * mempool_free_bulk - return elements to a mempool
621 * @pool: pointer to the memory pool
622 * @elems: elements to return
623 * @count: number of elements to return
624 *
625 * Returns a number of elements from the start of @elem to @pool if @pool needs
626 * replenishing and sets their slots in @elem to NULL. Other elements are left
627 * in @elem.
628 *
629 * Return: number of elements transferred to @pool. Elements are always
630 * transferred from the beginning of @elem, so the return value can be used as
631 * an offset into @elem for the freeing the remaining elements in the caller.
632 */
633unsigned int mempool_free_bulk(struct mempool *pool, void **elems,
634 unsigned int count)
635{
636 unsigned long flags;
637 unsigned int freed = 0;
638 bool added = false;
639
640 /*
641 * Paired with the wmb in mempool_alloc(). The preceding read is
642 * for @element and the following @pool->curr_nr. This ensures
643 * that the visible value of @pool->curr_nr is from after the
644 * allocation of @element. This is necessary for fringe cases
645 * where @element was passed to this task without going through
646 * barriers.
647 *
648 * For example, assume @p is %NULL at the beginning and one task
649 * performs "p = mempool_alloc(...);" while another task is doing
650 * "while (!p) cpu_relax(); mempool_free(p, ...);". This function
651 * may end up using curr_nr value which is from before allocation
652 * of @p without the following rmb.
653 */
654 smp_rmb();
655
656 /*
657 * For correctness, we need a test which is guaranteed to trigger
658 * if curr_nr + #allocated == min_nr. Testing curr_nr < min_nr
659 * without locking achieves that and refilling as soon as possible
660 * is desirable.
661 *
662 * Because curr_nr visible here is always a value after the
663 * allocation of @element, any task which decremented curr_nr below
664 * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
665 * incremented to min_nr afterwards. If curr_nr gets incremented
666 * to min_nr after the allocation of @element, the elements
667 * allocated after that are subject to the same guarantee.
668 *
669 * Waiters happen iff curr_nr is 0 and the above guarantee also
670 * ensures that there will be frees which return elements to the
671 * pool waking up the waiters.
672 *
673 * For zero-minimum pools, curr_nr < min_nr (0 < 0) never succeeds,
674 * so waiters sleeping on pool->wait would never be woken by the
675 * wake-up path of previous test. This explicit check ensures the
676 * allocation of element when both min_nr and curr_nr are 0, and
677 * any active waiters are properly awakened.
678 */
679 if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
680 spin_lock_irqsave(&pool->lock, flags);
681 while (pool->curr_nr < pool->min_nr && freed < count) {
682 add_element(pool, elems[freed++]);
683 added = true;
684 }
685 spin_unlock_irqrestore(&pool->lock, flags);
686 } else if (unlikely(pool->min_nr == 0 &&
687 READ_ONCE(pool->curr_nr) == 0)) {
688 /* Handle the min_nr = 0 edge case: */
689 spin_lock_irqsave(&pool->lock, flags);
690 if (likely(pool->curr_nr == 0)) {
691 add_element(pool, elems[freed++]);
692 added = true;
693 }
694 spin_unlock_irqrestore(&pool->lock, flags);
695 }
696
697 if (unlikely(added) && wq_has_sleeper(&pool->wait))
698 wake_up(&pool->wait);
699
700 return freed;
701}
702EXPORT_SYMBOL_GPL(mempool_free_bulk);
703
704/**
705 * mempool_free - return an element to the pool.
706 * @element: element to return
707 * @pool: pointer to the memory pool
708 *
709 * Returns @element to @pool if it needs replenishing, else frees it using
710 * the free_fn callback in @pool.
711 *
712 * This function only sleeps if the free_fn callback sleeps.
713 */
714void mempool_free(void *element, struct mempool *pool)
715{
716 if (likely(element) && !mempool_free_bulk(pool, &element, 1))
717 pool->free(element, pool->pool_data);
718}
719EXPORT_SYMBOL(mempool_free);
720
721/*
722 * A commonly used alloc and free fn.
723 */
724void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
725{
726 struct kmem_cache *mem = pool_data;
727 VM_BUG_ON(mem->ctor);
728 return kmem_cache_alloc_noprof(mem, gfp_mask);
729}
730EXPORT_SYMBOL(mempool_alloc_slab);
731
732void mempool_free_slab(void *element, void *pool_data)
733{
734 struct kmem_cache *mem = pool_data;
735 kmem_cache_free(mem, element);
736}
737EXPORT_SYMBOL(mempool_free_slab);
738
739/*
740 * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
741 * specified by pool_data
742 */
743void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
744{
745 size_t size = (size_t)pool_data;
746 return kmalloc_noprof(size, gfp_mask);
747}
748EXPORT_SYMBOL(mempool_kmalloc);
749
750void mempool_kfree(void *element, void *pool_data)
751{
752 kfree(element);
753}
754EXPORT_SYMBOL(mempool_kfree);
755
756/*
757 * A simple mempool-backed page allocator that allocates pages
758 * of the order specified by pool_data.
759 */
760void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
761{
762 int order = (int)(long)pool_data;
763 return alloc_pages_noprof(gfp_mask, order);
764}
765EXPORT_SYMBOL(mempool_alloc_pages);
766
767void mempool_free_pages(void *element, void *pool_data)
768{
769 int order = (int)(long)pool_data;
770 __free_pages(element, order);
771}
772EXPORT_SYMBOL(mempool_free_pages);