projects/linux·blob master · refs

824 lines · 21.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/memcontrol.h>
3#include <linux/rwsem.h>
4#include <linux/shrinker.h>
5#include <linux/rculist.h>
6#include <trace/events/vmscan.h>
7
8#include "internal.h"
9
10LIST_HEAD(shrinker_list);
11DEFINE_MUTEX(shrinker_mutex);
12
13#ifdef CONFIG_MEMCG
14static int shrinker_nr_max;
15
16static inline int shrinker_unit_size(int nr_items)
17{
18 return (DIV_ROUND_UP(nr_items, SHRINKER_UNIT_BITS) * sizeof(struct shrinker_info_unit *));
19}
20
21static inline void shrinker_unit_free(struct shrinker_info *info, int start)
22{
23 struct shrinker_info_unit **unit;
24 int nr, i;
25
26 if (!info)
27 return;
28
29 unit = info->unit;
30 nr = DIV_ROUND_UP(info->map_nr_max, SHRINKER_UNIT_BITS);
31
32 for (i = start; i < nr; i++) {
33 if (!unit[i])
34 break;
35
36 kfree(unit[i]);
37 unit[i] = NULL;
38 }
39}
40
41static inline int shrinker_unit_alloc(struct shrinker_info *new,
42 struct shrinker_info *old, int nid)
43{
44 struct shrinker_info_unit *unit;
45 int nr = DIV_ROUND_UP(new->map_nr_max, SHRINKER_UNIT_BITS);
46 int start = old ? DIV_ROUND_UP(old->map_nr_max, SHRINKER_UNIT_BITS) : 0;
47 int i;
48
49 for (i = start; i < nr; i++) {
50 unit = kzalloc_node(sizeof(*unit), GFP_KERNEL, nid);
51 if (!unit) {
52 shrinker_unit_free(new, start);
53 return -ENOMEM;
54 }
55
56 new->unit[i] = unit;
57 }
58
59 return 0;
60}
61
62static void __free_shrinker_info(struct mem_cgroup *memcg)
63{
64 struct mem_cgroup_per_node *pn;
65 struct shrinker_info *info;
66 int nid;
67
68 lockdep_assert_held(&shrinker_mutex);
69
70 for_each_node(nid) {
71 pn = memcg->nodeinfo[nid];
72 info = rcu_dereference_protected(pn->shrinker_info, true);
73 shrinker_unit_free(info, 0);
74 kvfree(info);
75 rcu_assign_pointer(pn->shrinker_info, NULL);
76 }
77}
78
79void free_shrinker_info(struct mem_cgroup *memcg)
80{
81 mutex_lock(&shrinker_mutex);
82 __free_shrinker_info(memcg);
83 mutex_unlock(&shrinker_mutex);
84}
85
86int alloc_shrinker_info(struct mem_cgroup *memcg)
87{
88 int nid, ret = 0;
89 int array_size = 0;
90
91 mutex_lock(&shrinker_mutex);
92 array_size = shrinker_unit_size(shrinker_nr_max);
93 for_each_node(nid) {
94 struct shrinker_info *info = kvzalloc_node(sizeof(*info) + array_size,
95 GFP_KERNEL, nid);
96 if (!info)
97 goto err;
98 info->map_nr_max = shrinker_nr_max;
99 if (shrinker_unit_alloc(info, NULL, nid)) {
100 kvfree(info);
101 goto err;
102 }
103 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
104 }
105 mutex_unlock(&shrinker_mutex);
106
107 return ret;
108
109err:
110 __free_shrinker_info(memcg);
111 mutex_unlock(&shrinker_mutex);
112 return -ENOMEM;
113}
114
115static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
116 int nid)
117{
118 return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
119 lockdep_is_held(&shrinker_mutex));
120}
121
122static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size,
123 int old_size, int new_nr_max)
124{
125 struct shrinker_info *new, *old;
126 struct mem_cgroup_per_node *pn;
127 int nid;
128
129 for_each_node(nid) {
130 pn = memcg->nodeinfo[nid];
131 old = shrinker_info_protected(memcg, nid);
132 /* Not yet online memcg */
133 if (!old)
134 return 0;
135
136 /* Already expanded this shrinker_info */
137 if (new_nr_max <= old->map_nr_max)
138 continue;
139
140 new = kvzalloc_node(sizeof(*new) + new_size, GFP_KERNEL, nid);
141 if (!new)
142 return -ENOMEM;
143
144 new->map_nr_max = new_nr_max;
145
146 memcpy(new->unit, old->unit, old_size);
147 if (shrinker_unit_alloc(new, old, nid)) {
148 kvfree(new);
149 return -ENOMEM;
150 }
151
152 rcu_assign_pointer(pn->shrinker_info, new);
153 kvfree_rcu(old, rcu);
154 }
155
156 return 0;
157}
158
159static int expand_shrinker_info(int new_id)
160{
161 int ret = 0;
162 int new_nr_max = round_up(new_id + 1, SHRINKER_UNIT_BITS);
163 int new_size, old_size = 0;
164 struct mem_cgroup *memcg;
165
166 if (!root_mem_cgroup)
167 goto out;
168
169 lockdep_assert_held(&shrinker_mutex);
170
171 new_size = shrinker_unit_size(new_nr_max);
172 old_size = shrinker_unit_size(shrinker_nr_max);
173
174 memcg = mem_cgroup_iter(NULL, NULL, NULL);
175 do {
176 ret = expand_one_shrinker_info(memcg, new_size, old_size,
177 new_nr_max);
178 if (ret) {
179 mem_cgroup_iter_break(NULL, memcg);
180 goto out;
181 }
182 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
183out:
184 if (!ret)
185 shrinker_nr_max = new_nr_max;
186
187 return ret;
188}
189
190static inline int shrinker_id_to_index(int shrinker_id)
191{
192 return shrinker_id / SHRINKER_UNIT_BITS;
193}
194
195static inline int shrinker_id_to_offset(int shrinker_id)
196{
197 return shrinker_id % SHRINKER_UNIT_BITS;
198}
199
200static inline int calc_shrinker_id(int index, int offset)
201{
202 return index * SHRINKER_UNIT_BITS + offset;
203}
204
205void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
206{
207 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
208 struct shrinker_info *info;
209
210 rcu_read_lock();
211 info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
212 if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
213 struct shrinker_info_unit *unit;
214
215 unit = info->unit[shrinker_id_to_index(shrinker_id)];
216 /* Pairs with smp mb in shrink_slab() */
217 smp_mb__before_atomic();
218 set_bit(shrinker_id_to_offset(shrinker_id), unit->map);
219 }
220 rcu_read_unlock();
221 }
222}
223
224static DEFINE_IDR(shrinker_idr);
225
226static int shrinker_memcg_alloc(struct shrinker *shrinker)
227{
228 int id;
229
230 shrinker->id = -1;
231
232 if (mem_cgroup_disabled())
233 return -ENOSYS;
234 if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB))
235 return -ENOSYS;
236
237 guard(mutex)(&shrinker_mutex);
238 id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
239 if (id < 0)
240 return id;
241
242 if (id >= shrinker_nr_max) {
243 if (expand_shrinker_info(id)) {
244 idr_remove(&shrinker_idr, id);
245 return -ENOMEM;
246 }
247 }
248 shrinker->id = id;
249 return 0;
250}
251
252static void shrinker_memcg_remove(struct shrinker *shrinker)
253{
254 int id = shrinker->id;
255
256 BUG_ON(id < 0);
257
258 lockdep_assert_held(&shrinker_mutex);
259
260 idr_remove(&shrinker_idr, id);
261}
262
263static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
264 struct mem_cgroup *memcg)
265{
266 struct shrinker_info *info;
267 struct shrinker_info_unit *unit;
268 long nr_deferred;
269
270 rcu_read_lock();
271 info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
272 unit = info->unit[shrinker_id_to_index(shrinker->id)];
273 nr_deferred = atomic_long_xchg(&unit->nr_deferred[shrinker_id_to_offset(shrinker->id)], 0);
274 rcu_read_unlock();
275
276 return nr_deferred;
277}
278
279static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
280 struct mem_cgroup *memcg)
281{
282 struct shrinker_info *info;
283 struct shrinker_info_unit *unit;
284 long nr_deferred;
285
286 rcu_read_lock();
287 info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
288 unit = info->unit[shrinker_id_to_index(shrinker->id)];
289 nr_deferred =
290 atomic_long_add_return(nr, &unit->nr_deferred[shrinker_id_to_offset(shrinker->id)]);
291 rcu_read_unlock();
292
293 return nr_deferred;
294}
295
296void reparent_shrinker_deferred(struct mem_cgroup *memcg)
297{
298 int nid, index, offset;
299 long nr;
300 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
301 struct shrinker_info *child_info, *parent_info;
302 struct shrinker_info_unit *child_unit, *parent_unit;
303
304 /* Prevent from concurrent shrinker_info expand */
305 mutex_lock(&shrinker_mutex);
306 for_each_node(nid) {
307 child_info = shrinker_info_protected(memcg, nid);
308 parent_info = shrinker_info_protected(parent, nid);
309 for (index = 0; index < shrinker_id_to_index(child_info->map_nr_max); index++) {
310 child_unit = child_info->unit[index];
311 parent_unit = parent_info->unit[index];
312 for (offset = 0; offset < SHRINKER_UNIT_BITS; offset++) {
313 nr = atomic_long_read(&child_unit->nr_deferred[offset]);
314 atomic_long_add(nr, &parent_unit->nr_deferred[offset]);
315 }
316 }
317 }
318 mutex_unlock(&shrinker_mutex);
319}
320#else
321static int shrinker_memcg_alloc(struct shrinker *shrinker)
322{
323 return -ENOSYS;
324}
325
326static void shrinker_memcg_remove(struct shrinker *shrinker)
327{
328}
329
330static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
331 struct mem_cgroup *memcg)
332{
333 return 0;
334}
335
336static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
337 struct mem_cgroup *memcg)
338{
339 return 0;
340}
341#endif /* CONFIG_MEMCG */
342
343static long xchg_nr_deferred(struct shrinker *shrinker,
344 struct shrink_control *sc)
345{
346 int nid = sc->nid;
347
348 if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
349 nid = 0;
350
351 if (sc->memcg &&
352 (shrinker->flags & SHRINKER_MEMCG_AWARE))
353 return xchg_nr_deferred_memcg(nid, shrinker,
354 sc->memcg);
355
356 return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
357}
358
359
360static long add_nr_deferred(long nr, struct shrinker *shrinker,
361 struct shrink_control *sc)
362{
363 int nid = sc->nid;
364
365 if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
366 nid = 0;
367
368 if (sc->memcg &&
369 (shrinker->flags & SHRINKER_MEMCG_AWARE))
370 return add_nr_deferred_memcg(nr, nid, shrinker,
371 sc->memcg);
372
373 return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
374}
375
376#define SHRINK_BATCH 128
377
378static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
379 struct shrinker *shrinker, int priority)
380{
381 unsigned long freed = 0;
382 unsigned long long delta;
383 long total_scan;
384 long freeable;
385 long nr;
386 long new_nr;
387 long batch_size = shrinker->batch ? shrinker->batch
388 : SHRINK_BATCH;
389 long scanned = 0, next_deferred;
390
391 freeable = shrinker->count_objects(shrinker, shrinkctl);
392 if (freeable == 0 || freeable == SHRINK_EMPTY)
393 return freeable;
394
395 /*
396 * copy the current shrinker scan count into a local variable
397 * and zero it so that other concurrent shrinker invocations
398 * don't also do this scanning work.
399 */
400 nr = xchg_nr_deferred(shrinker, shrinkctl);
401
402 if (shrinker->seeks) {
403 delta = freeable >> priority;
404 delta *= 4;
405 do_div(delta, shrinker->seeks);
406 } else {
407 /*
408 * These objects don't require any IO to create. Trim
409 * them aggressively under memory pressure to keep
410 * them from causing refetches in the IO caches.
411 */
412 delta = freeable / 2;
413 }
414
415 total_scan = nr >> priority;
416 total_scan += delta;
417 total_scan = min(total_scan, (2 * freeable));
418
419 trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
420 freeable, delta, total_scan, priority,
421 shrinkctl->memcg);
422
423 /*
424 * Normally, we should not scan less than batch_size objects in one
425 * pass to avoid too frequent shrinker calls, but if the slab has less
426 * than batch_size objects in total and we are really tight on memory,
427 * we will try to reclaim all available objects, otherwise we can end
428 * up failing allocations although there are plenty of reclaimable
429 * objects spread over several slabs with usage less than the
430 * batch_size.
431 *
432 * We detect the "tight on memory" situations by looking at the total
433 * number of objects we want to scan (total_scan). If it is greater
434 * than the total number of objects on slab (freeable), we must be
435 * scanning at high prio and therefore should try to reclaim as much as
436 * possible.
437 */
438 while (total_scan >= batch_size ||
439 total_scan >= freeable) {
440 unsigned long ret;
441 unsigned long nr_to_scan = min(batch_size, total_scan);
442
443 shrinkctl->nr_to_scan = nr_to_scan;
444 shrinkctl->nr_scanned = nr_to_scan;
445 ret = shrinker->scan_objects(shrinker, shrinkctl);
446 if (ret == SHRINK_STOP)
447 break;
448 freed += ret;
449
450 count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
451 total_scan -= shrinkctl->nr_scanned;
452 scanned += shrinkctl->nr_scanned;
453
454 cond_resched();
455 }
456
457 /*
458 * The deferred work is increased by any new work (delta) that wasn't
459 * done, decreased by old deferred work that was done now.
460 *
461 * And it is capped to two times of the freeable items.
462 */
463 next_deferred = max_t(long, (nr + delta - scanned), 0);
464 next_deferred = min(next_deferred, (2 * freeable));
465
466 /*
467 * move the unused scan count back into the shrinker in a
468 * manner that handles concurrent updates.
469 */
470 new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);
471
472 trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan,
473 shrinkctl->memcg);
474 return freed;
475}
476
477#ifdef CONFIG_MEMCG
478static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
479 struct mem_cgroup *memcg, int priority)
480{
481 struct shrinker_info *info;
482 unsigned long ret, freed = 0;
483 int offset, index = 0;
484
485 if (!mem_cgroup_online(memcg))
486 return 0;
487
488 /*
489 * lockless algorithm of memcg shrink.
490 *
491 * The shrinker_info may be freed asynchronously via RCU in the
492 * expand_one_shrinker_info(), so the rcu_read_lock() needs to be used
493 * to ensure the existence of the shrinker_info.
494 *
495 * The shrinker_info_unit is never freed unless its corresponding memcg
496 * is destroyed. Here we already hold the refcount of memcg, so the
497 * memcg will not be destroyed, and of course shrinker_info_unit will
498 * not be freed.
499 *
500 * So in the memcg shrink:
501 * step 1: use rcu_read_lock() to guarantee existence of the
502 * shrinker_info.
503 * step 2: after getting shrinker_info_unit we can safely release the
504 * RCU lock.
505 * step 3: traverse the bitmap and calculate shrinker_id
506 * step 4: use rcu_read_lock() to guarantee existence of the shrinker.
507 * step 5: use shrinker_id to find the shrinker, then use
508 * shrinker_try_get() to guarantee existence of the shrinker,
509 * then we can release the RCU lock to do do_shrink_slab() that
510 * may sleep.
511 * step 6: do shrinker_put() paired with step 5 to put the refcount,
512 * if the refcount reaches 0, then wake up the waiter in
513 * shrinker_free() by calling complete().
514 * Note: here is different from the global shrink, we don't
515 * need to acquire the RCU lock to guarantee existence of
516 * the shrinker, because we don't need to use this
517 * shrinker to traverse the next shrinker in the bitmap.
518 * step 7: we have already exited the read-side of rcu critical section
519 * before calling do_shrink_slab(), the shrinker_info may be
520 * released in expand_one_shrinker_info(), so go back to step 1
521 * to reacquire the shrinker_info.
522 */
523again:
524 rcu_read_lock();
525 info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
526 if (unlikely(!info))
527 goto unlock;
528
529 if (index < shrinker_id_to_index(info->map_nr_max)) {
530 struct shrinker_info_unit *unit;
531
532 unit = info->unit[index];
533
534 rcu_read_unlock();
535
536 for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) {
537 struct shrink_control sc = {
538 .gfp_mask = gfp_mask,
539 .nid = nid,
540 .memcg = memcg,
541 };
542 struct shrinker *shrinker;
543 int shrinker_id = calc_shrinker_id(index, offset);
544
545 rcu_read_lock();
546 shrinker = idr_find(&shrinker_idr, shrinker_id);
547 if (unlikely(!shrinker || !shrinker_try_get(shrinker))) {
548 clear_bit(offset, unit->map);
549 rcu_read_unlock();
550 continue;
551 }
552 rcu_read_unlock();
553
554 /* Call non-slab shrinkers even though kmem is disabled */
555 if (!memcg_kmem_online() &&
556 !(shrinker->flags & SHRINKER_NONSLAB)) {
557 clear_bit(offset, unit->map);
558 shrinker_put(shrinker);
559 continue;
560 }
561
562 ret = do_shrink_slab(&sc, shrinker, priority);
563 if (ret == SHRINK_EMPTY) {
564 clear_bit(offset, unit->map);
565 /*
566 * After the shrinker reported that it had no objects to
567 * free, but before we cleared the corresponding bit in
568 * the memcg shrinker map, a new object might have been
569 * added. To make sure, we have the bit set in this
570 * case, we invoke the shrinker one more time and reset
571 * the bit if it reports that it is not empty anymore.
572 * The memory barrier here pairs with the barrier in
573 * set_shrinker_bit():
574 *
575 * list_lru_add() shrink_slab_memcg()
576 * list_add_tail() clear_bit()
577 * <MB> <MB>
578 * set_bit() do_shrink_slab()
579 */
580 smp_mb__after_atomic();
581 ret = do_shrink_slab(&sc, shrinker, priority);
582 if (ret == SHRINK_EMPTY)
583 ret = 0;
584 else
585 set_shrinker_bit(memcg, nid, shrinker_id);
586 }
587 freed += ret;
588 shrinker_put(shrinker);
589 }
590
591 index++;
592 goto again;
593 }
594unlock:
595 rcu_read_unlock();
596 return freed;
597}
598#else /* !CONFIG_MEMCG */
599static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
600 struct mem_cgroup *memcg, int priority)
601{
602 return 0;
603}
604#endif /* CONFIG_MEMCG */
605
606/**
607 * shrink_slab - shrink slab caches
608 * @gfp_mask: allocation context
609 * @nid: node whose slab caches to target
610 * @memcg: memory cgroup whose slab caches to target
611 * @priority: the reclaim priority
612 *
613 * Call the shrink functions to age shrinkable caches.
614 *
615 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
616 * unaware shrinkers will receive a node id of 0 instead.
617 *
618 * @memcg specifies the memory cgroup to target. Unaware shrinkers
619 * are called only if it is the root cgroup.
620 *
621 * @priority is sc->priority, we take the number of objects and >> by priority
622 * in order to get the scan target.
623 *
624 * Returns the number of reclaimed slab objects.
625 */
626unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
627 int priority)
628{
629 unsigned long ret, freed = 0;
630 struct shrinker *shrinker;
631
632 /*
633 * The root memcg might be allocated even though memcg is disabled
634 * via "cgroup_disable=memory" boot parameter. This could make
635 * mem_cgroup_is_root() return false, then just run memcg slab
636 * shrink, but skip global shrink. This may result in premature
637 * oom.
638 */
639 if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
640 return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
641
642 /*
643 * lockless algorithm of global shrink.
644 *
645 * In the unregistration setp, the shrinker will be freed asynchronously
646 * via RCU after its refcount reaches 0. So both rcu_read_lock() and
647 * shrinker_try_get() can be used to ensure the existence of the shrinker.
648 *
649 * So in the global shrink:
650 * step 1: use rcu_read_lock() to guarantee existence of the shrinker
651 * and the validity of the shrinker_list walk.
652 * step 2: use shrinker_try_get() to try get the refcount, if successful,
653 * then the existence of the shrinker can also be guaranteed,
654 * so we can release the RCU lock to do do_shrink_slab() that
655 * may sleep.
656 * step 3: *MUST* to reacquire the RCU lock before calling shrinker_put(),
657 * which ensures that neither this shrinker nor the next shrinker
658 * will be freed in the next traversal operation.
659 * step 4: do shrinker_put() paired with step 2 to put the refcount,
660 * if the refcount reaches 0, then wake up the waiter in
661 * shrinker_free() by calling complete().
662 */
663 rcu_read_lock();
664 list_for_each_entry_rcu(shrinker, &shrinker_list, list) {
665 struct shrink_control sc = {
666 .gfp_mask = gfp_mask,
667 .nid = nid,
668 .memcg = memcg,
669 };
670
671 if (!shrinker_try_get(shrinker))
672 continue;
673
674 rcu_read_unlock();
675
676 ret = do_shrink_slab(&sc, shrinker, priority);
677 if (ret == SHRINK_EMPTY)
678 ret = 0;
679 freed += ret;
680
681 rcu_read_lock();
682 shrinker_put(shrinker);
683 }
684
685 rcu_read_unlock();
686 cond_resched();
687 return freed;
688}
689
690struct shrinker *shrinker_alloc(unsigned int flags, const char *fmt, ...)
691{
692 struct shrinker *shrinker;
693 unsigned int size;
694 va_list ap;
695 int err;
696
697 shrinker = kzalloc_obj(struct shrinker);
698 if (!shrinker)
699 return NULL;
700
701 va_start(ap, fmt);
702 err = shrinker_debugfs_name_alloc(shrinker, fmt, ap);
703 va_end(ap);
704 if (err)
705 goto err_name;
706
707 shrinker->flags = flags | SHRINKER_ALLOCATED;
708 shrinker->seeks = DEFAULT_SEEKS;
709
710 if (flags & SHRINKER_MEMCG_AWARE) {
711 err = shrinker_memcg_alloc(shrinker);
712 if (err == -ENOSYS) {
713 /* Memcg is not supported, fallback to non-memcg-aware shrinker. */
714 shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
715 goto non_memcg;
716 }
717
718 if (err)
719 goto err_flags;
720
721 return shrinker;
722 }
723
724non_memcg:
725 /*
726 * The nr_deferred is available on per memcg level for memcg aware
727 * shrinkers, so only allocate nr_deferred in the following cases:
728 * - non-memcg-aware shrinkers
729 * - !CONFIG_MEMCG
730 * - memcg is disabled by kernel command line
731 * - non-slab shrinkers: when memcg kmem is disabled
732 */
733 size = sizeof(*shrinker->nr_deferred);
734 if (flags & SHRINKER_NUMA_AWARE)
735 size *= nr_node_ids;
736
737 shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
738 if (!shrinker->nr_deferred)
739 goto err_flags;
740
741 return shrinker;
742
743err_flags:
744 shrinker_debugfs_name_free(shrinker);
745err_name:
746 kfree(shrinker);
747 return NULL;
748}
749EXPORT_SYMBOL_GPL(shrinker_alloc);
750
751void shrinker_register(struct shrinker *shrinker)
752{
753 if (unlikely(!(shrinker->flags & SHRINKER_ALLOCATED))) {
754 pr_warn("Must use shrinker_alloc() to dynamically allocate the shrinker");
755 return;
756 }
757
758 mutex_lock(&shrinker_mutex);
759 list_add_tail_rcu(&shrinker->list, &shrinker_list);
760 shrinker->flags |= SHRINKER_REGISTERED;
761 shrinker_debugfs_add(shrinker);
762 mutex_unlock(&shrinker_mutex);
763
764 init_completion(&shrinker->done);
765 /*
766 * Now the shrinker is fully set up, take the first reference to it to
767 * indicate that lookup operations are now allowed to use it via
768 * shrinker_try_get().
769 */
770 refcount_set(&shrinker->refcount, 1);
771}
772EXPORT_SYMBOL_GPL(shrinker_register);
773
774static void shrinker_free_rcu_cb(struct rcu_head *head)
775{
776 struct shrinker *shrinker = container_of(head, struct shrinker, rcu);
777
778 kfree(shrinker->nr_deferred);
779 kfree(shrinker);
780}
781
782void shrinker_free(struct shrinker *shrinker)
783{
784 struct dentry *debugfs_entry = NULL;
785 int debugfs_id;
786
787 if (!shrinker)
788 return;
789
790 if (shrinker->flags & SHRINKER_REGISTERED) {
791 /* drop the initial refcount */
792 shrinker_put(shrinker);
793 /*
794 * Wait for all lookups of the shrinker to complete, after that,
795 * no shrinker is running or will run again, then we can safely
796 * free it asynchronously via RCU and safely free the structure
797 * where the shrinker is located, such as super_block etc.
798 */
799 wait_for_completion(&shrinker->done);
800 }
801
802 mutex_lock(&shrinker_mutex);
803 if (shrinker->flags & SHRINKER_REGISTERED) {
804 /*
805 * Now we can safely remove it from the shrinker_list and then
806 * free it.
807 */
808 list_del_rcu(&shrinker->list);
809 debugfs_entry = shrinker_debugfs_detach(shrinker, &debugfs_id);
810 shrinker->flags &= ~SHRINKER_REGISTERED;
811 }
812
813 shrinker_debugfs_name_free(shrinker);
814
815 if (shrinker->flags & SHRINKER_MEMCG_AWARE)
816 shrinker_memcg_remove(shrinker);
817 mutex_unlock(&shrinker_mutex);
818
819 if (debugfs_entry)
820 shrinker_debugfs_remove(debugfs_entry, debugfs_id);
821
822 call_rcu(&shrinker->rcu, shrinker_free_rcu_cb);
823}
824EXPORT_SYMBOL_GPL(shrinker_free);