projects/linux·blob master · refs

1044 lines · 26.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2023-2024 Intel Corporation (Maarten Lankhorst <[email protected]>)
4 * Copyright 2024 Red Hat (Maxime Ripard <[email protected]>)
5 * Partially based on the rdma and misc controllers, which bear the following copyrights:
6 *
7 * Copyright 2020 Google LLC
8 * Copyright (C) 2016 Parav Pandit <[email protected]>
9 */
10
11#include <linux/cgroup.h>
12#include <linux/cgroup_dmem.h>
13#include <linux/list.h>
14#include <linux/mutex.h>
15#include <linux/page_counter.h>
16#include <linux/parser.h>
17#include <linux/refcount.h>
18#include <linux/rculist.h>
19#include <linux/slab.h>
20#include <linux/srcu.h>
21
22/* Maximum reclaim attempts before giving up when lowering dmem.max. */
23#define DMEM_MAX_RECLAIM_RETRIES 16
24
25/* SRCU domain serialising reclaim callbacks against region unregistration. */
26DEFINE_STATIC_SRCU(dmemcg_srcu);
27
28struct dmem_cgroup_region {
29 /**
30 * @ref: References keeping the region alive.
31 * Keeps the region reference alive after a succesful RCU lookup.
32 */
33 struct kref ref;
34
35 /** @rcu: RCU head for freeing */
36 struct rcu_head rcu;
37
38 /**
39 * @region_node: Linked into &dmem_cgroup_regions list.
40 * Protected by RCU and global spinlock.
41 */
42 struct list_head region_node;
43
44 /**
45 * @pools: List of pools linked to this region.
46 * Protected by global spinlock only
47 */
48 struct list_head pools;
49
50 /** @size: Size of region, in bytes */
51 u64 size;
52
53 /** @name: Name describing the node, set by dmem_cgroup_register_region */
54 char *name;
55
56 /**
57 * @unregistered: Whether the region is unregistered by its caller.
58 * No new pools should be added to the region afterwards, and no new
59 * reclaim callbacks should be invoked.
60 */
61 bool unregistered;
62
63 /**
64 * @ops: Optional driver operations for this region.
65 */
66 const struct dmem_cgroup_ops *ops;
67
68 /** @reclaim_priv: Private data passed to @ops->reclaim. */
69 void *reclaim_priv;
70};
71
72struct dmemcg_state {
73 struct cgroup_subsys_state css;
74
75 struct list_head pools;
76};
77
78struct dmem_cgroup_pool_state {
79 struct dmem_cgroup_region *region;
80 struct dmemcg_state *cs;
81
82 /* css node, RCU protected against region teardown */
83 struct list_head css_node;
84
85 /* dev node, no RCU protection required */
86 struct list_head region_node;
87
88 struct rcu_head rcu;
89
90 struct page_counter cnt;
91 struct dmem_cgroup_pool_state *parent;
92
93 refcount_t ref;
94 bool inited;
95};
96
97/*
98 * 3 operations require locking protection:
99 * - Registering and unregistering region to/from list, requires global lock.
100 * - Adding a dmem_cgroup_pool_state to a CSS, removing when CSS is freed.
101 * - Adding a dmem_cgroup_pool_state to a region list.
102 *
103 * Since for the most common operations RCU provides enough protection, I
104 * do not think more granular locking makes sense. Most protection is offered
105 * by RCU and the lockless operating page_counter.
106 */
107static DEFINE_SPINLOCK(dmemcg_lock);
108static LIST_HEAD(dmem_cgroup_regions);
109
110static void dmemcg_free_region(struct kref *ref);
111static void dmemcg_pool_free_rcu(struct rcu_head *rcu);
112
113static inline struct dmemcg_state *
114css_to_dmemcs(struct cgroup_subsys_state *css)
115{
116 return container_of(css, struct dmemcg_state, css);
117}
118
119static inline struct dmemcg_state *get_current_dmemcs(void)
120{
121 return css_to_dmemcs(task_get_css(current, dmem_cgrp_id));
122}
123
124static struct dmemcg_state *parent_dmemcs(struct dmemcg_state *cg)
125{
126 return cg->css.parent ? css_to_dmemcs(cg->css.parent) : NULL;
127}
128
129static void dmemcg_pool_get(struct dmem_cgroup_pool_state *pool)
130{
131 refcount_inc(&pool->ref);
132}
133
134static bool dmemcg_pool_tryget(struct dmem_cgroup_pool_state *pool)
135{
136 return refcount_inc_not_zero(&pool->ref);
137}
138
139static void dmemcg_pool_put(struct dmem_cgroup_pool_state *pool)
140{
141 if (!refcount_dec_and_test(&pool->ref))
142 return;
143
144 call_rcu(&pool->rcu, dmemcg_pool_free_rcu);
145}
146
147static void dmemcg_pool_free_rcu(struct rcu_head *rcu)
148{
149 struct dmem_cgroup_pool_state *pool = container_of(rcu, typeof(*pool), rcu);
150
151 if (pool->parent)
152 dmemcg_pool_put(pool->parent);
153 kref_put(&pool->region->ref, dmemcg_free_region);
154 kfree(pool);
155}
156
157static void free_cg_pool(struct dmem_cgroup_pool_state *pool)
158{
159 list_del(&pool->region_node);
160 dmemcg_pool_put(pool);
161}
162
163static void
164set_resource_min(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
165{
166 page_counter_set_min(&pool->cnt, val);
167}
168
169static void
170set_resource_low(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
171{
172 page_counter_set_low(&pool->cnt, val);
173}
174
175static void
176set_resource_max(struct dmem_cgroup_pool_state *pool, u64 val, bool nonblock)
177{
178 struct dmem_cgroup_region *region = pool->region;
179 unsigned long limit = (unsigned long)val;
180
181 /* Apply the new limit immediately so concurrent allocations are throttled. */
182 xchg(&pool->cnt.max, limit);
183
184 if (nonblock)
185 return;
186
187 int srcu_idx = srcu_read_lock(&dmemcg_srcu);
188
189 if (!READ_ONCE(region->unregistered) && region->ops && region->ops->reclaim) {
190 for (int retries = DMEM_MAX_RECLAIM_RETRIES; ; ) {
191 u64 usage = page_counter_read(&pool->cnt);
192 int ret;
193
194 if (usage <= limit)
195 break;
196
197 if (signal_pending(current))
198 break;
199
200 ret = region->ops->reclaim(pool, usage - limit, region->reclaim_priv);
201
202 /* -ENOSPC means no progress; other errors are fatal. */
203 if (ret && (ret != -ENOSPC || !retries--))
204 break;
205
206 cond_resched();
207 }
208 }
209 srcu_read_unlock(&dmemcg_srcu, srcu_idx);
210}
211
212static u64 get_resource_low(struct dmem_cgroup_pool_state *pool)
213{
214 return pool ? READ_ONCE(pool->cnt.low) : 0;
215}
216
217static u64 get_resource_min(struct dmem_cgroup_pool_state *pool)
218{
219 return pool ? READ_ONCE(pool->cnt.min) : 0;
220}
221
222static u64 get_resource_max(struct dmem_cgroup_pool_state *pool)
223{
224 return pool ? READ_ONCE(pool->cnt.max) : PAGE_COUNTER_MAX;
225}
226
227static u64 get_resource_current(struct dmem_cgroup_pool_state *pool)
228{
229 return pool ? page_counter_read(&pool->cnt) : 0;
230}
231
232static u64 get_resource_peak(struct dmem_cgroup_pool_state *pool)
233{
234 return pool ? READ_ONCE(pool->cnt.watermark) : 0;
235}
236
237static void reset_all_resource_limits(struct dmem_cgroup_pool_state *rpool)
238{
239 set_resource_min(rpool, 0, false);
240 set_resource_low(rpool, 0, false);
241 /* nonblock: raising to max makes reclaim a no-op; sleeping is forbidden here. */
242 set_resource_max(rpool, PAGE_COUNTER_MAX, true);
243}
244
245static void dmemcs_offline(struct cgroup_subsys_state *css)
246{
247 struct dmemcg_state *dmemcs = css_to_dmemcs(css);
248 struct dmem_cgroup_pool_state *pool;
249
250 rcu_read_lock();
251 list_for_each_entry_rcu(pool, &dmemcs->pools, css_node)
252 reset_all_resource_limits(pool);
253 rcu_read_unlock();
254}
255
256static void dmemcs_free(struct cgroup_subsys_state *css)
257{
258 struct dmemcg_state *dmemcs = css_to_dmemcs(css);
259 struct dmem_cgroup_pool_state *pool, *next;
260
261 spin_lock(&dmemcg_lock);
262 list_for_each_entry_safe(pool, next, &dmemcs->pools, css_node) {
263 /*
264 *The pool is dead and all references are 0,
265 * no need for RCU protection with list_del_rcu or freeing.
266 */
267 list_del(&pool->css_node);
268 free_cg_pool(pool);
269 }
270 spin_unlock(&dmemcg_lock);
271
272 kfree(dmemcs);
273}
274
275static struct cgroup_subsys_state *
276dmemcs_alloc(struct cgroup_subsys_state *parent_css)
277{
278 struct dmemcg_state *dmemcs = kzalloc_obj(*dmemcs);
279 if (!dmemcs)
280 return ERR_PTR(-ENOMEM);
281
282 INIT_LIST_HEAD(&dmemcs->pools);
283 return &dmemcs->css;
284}
285
286static struct dmem_cgroup_pool_state *
287find_cg_pool_locked(struct dmemcg_state *dmemcs, struct dmem_cgroup_region *region)
288{
289 struct dmem_cgroup_pool_state *pool;
290
291 list_for_each_entry_rcu(pool, &dmemcs->pools, css_node, spin_is_locked(&dmemcg_lock))
292 if (pool->region == region)
293 return pool;
294
295 return NULL;
296}
297
298static struct dmem_cgroup_pool_state *pool_parent(struct dmem_cgroup_pool_state *pool)
299{
300 if (!pool->cnt.parent)
301 return NULL;
302
303 return container_of(pool->cnt.parent, typeof(*pool), cnt);
304}
305
306static void
307dmem_cgroup_calculate_protection(struct dmem_cgroup_pool_state *limit_pool,
308 struct dmem_cgroup_pool_state *test_pool)
309{
310 struct page_counter *climit;
311 struct cgroup_subsys_state *css;
312 struct dmemcg_state *dmemcg_iter;
313 struct dmem_cgroup_pool_state *pool, *found_pool;
314
315 climit = &limit_pool->cnt;
316
317 rcu_read_lock();
318
319 css_for_each_descendant_pre(css, &limit_pool->cs->css) {
320 dmemcg_iter = container_of(css, struct dmemcg_state, css);
321 found_pool = NULL;
322
323 list_for_each_entry_rcu(pool, &dmemcg_iter->pools, css_node) {
324 if (pool->region == limit_pool->region) {
325 found_pool = pool;
326 break;
327 }
328 }
329 if (!found_pool)
330 continue;
331
332 page_counter_calculate_protection(
333 climit, &found_pool->cnt, true);
334
335 if (found_pool == test_pool)
336 break;
337 }
338 rcu_read_unlock();
339}
340
341/**
342 * dmem_cgroup_state_evict_valuable() - Check if we should evict from test_pool
343 * @limit_pool: The pool for which we hit limits
344 * @test_pool: The pool for which to test
345 * @ignore_low: Whether we have to respect low watermarks.
346 * @ret_hit_low: Pointer to whether it makes sense to consider low watermark.
347 *
348 * This function returns true if we can evict from @test_pool, false if not.
349 * When returning false and @ignore_low is false, @ret_hit_low may
350 * be set to true to indicate this function can be retried with @ignore_low
351 * set to true.
352 *
353 * Return: bool
354 */
355bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool,
356 struct dmem_cgroup_pool_state *test_pool,
357 bool ignore_low, bool *ret_hit_low)
358{
359 struct dmem_cgroup_pool_state *pool = test_pool;
360 struct page_counter *ctest;
361 u64 used, min, low;
362
363 /* Can always evict from current pool, despite limits */
364 if (limit_pool == test_pool)
365 return true;
366
367 if (limit_pool) {
368 if (!parent_dmemcs(limit_pool->cs))
369 return true;
370
371 for (pool = test_pool; pool && limit_pool != pool; pool = pool_parent(pool))
372 {}
373
374 if (!pool)
375 return false;
376 } else {
377 /*
378 * If there is no cgroup limiting memory usage, use the root
379 * cgroup instead for limit calculations.
380 */
381 for (limit_pool = test_pool; pool_parent(limit_pool); limit_pool = pool_parent(limit_pool))
382 {}
383 }
384
385 ctest = &test_pool->cnt;
386
387 dmem_cgroup_calculate_protection(limit_pool, test_pool);
388
389 used = page_counter_read(ctest);
390 min = READ_ONCE(ctest->emin);
391
392 if (used <= min)
393 return false;
394
395 if (!ignore_low) {
396 low = READ_ONCE(ctest->elow);
397 if (used > low)
398 return true;
399
400 *ret_hit_low = true;
401 return false;
402 }
403 return true;
404}
405EXPORT_SYMBOL_GPL(dmem_cgroup_state_evict_valuable);
406
407static struct dmem_cgroup_pool_state *
408alloc_pool_single(struct dmemcg_state *dmemcs, struct dmem_cgroup_region *region,
409 struct dmem_cgroup_pool_state **allocpool)
410{
411 struct dmemcg_state *parent = parent_dmemcs(dmemcs);
412 struct dmem_cgroup_pool_state *pool, *ppool = NULL;
413
414 if (!*allocpool) {
415 pool = kzalloc_obj(*pool, GFP_NOWAIT);
416 if (!pool)
417 return ERR_PTR(-ENOMEM);
418 } else {
419 pool = *allocpool;
420 *allocpool = NULL;
421 }
422
423 pool->region = region;
424 pool->cs = dmemcs;
425
426 if (parent)
427 ppool = find_cg_pool_locked(parent, region);
428
429 page_counter_init(&pool->cnt,
430 ppool ? &ppool->cnt : NULL, true);
431 reset_all_resource_limits(pool);
432 refcount_set(&pool->ref, 1);
433 kref_get(&region->ref);
434 if (ppool && !pool->parent) {
435 pool->parent = ppool;
436 dmemcg_pool_get(ppool);
437 }
438
439 list_add_tail_rcu(&pool->css_node, &dmemcs->pools);
440 list_add_tail(&pool->region_node, &region->pools);
441
442 if (!parent)
443 pool->inited = true;
444 else
445 pool->inited = ppool ? ppool->inited : false;
446 return pool;
447}
448
449static struct dmem_cgroup_pool_state *
450get_cg_pool_locked(struct dmemcg_state *dmemcs, struct dmem_cgroup_region *region,
451 struct dmem_cgroup_pool_state **allocpool)
452{
453 struct dmem_cgroup_pool_state *pool, *ppool, *retpool;
454 struct dmemcg_state *p, *pp;
455
456 /*
457 * Recursively create pool, we may not initialize yet on
458 * recursion, this is done as a separate step.
459 */
460 for (p = dmemcs; p; p = parent_dmemcs(p)) {
461 pool = find_cg_pool_locked(p, region);
462 if (!pool)
463 pool = alloc_pool_single(p, region, allocpool);
464
465 if (IS_ERR(pool))
466 return pool;
467
468 if (p == dmemcs && pool->inited)
469 return pool;
470
471 if (pool->inited)
472 break;
473 }
474
475 retpool = pool = find_cg_pool_locked(dmemcs, region);
476 for (p = dmemcs, pp = parent_dmemcs(dmemcs); pp; p = pp, pp = parent_dmemcs(p)) {
477 if (pool->inited)
478 break;
479
480 /* ppool was created if it didn't exist by above loop. */
481 ppool = find_cg_pool_locked(pp, region);
482
483 /* Fix up parent links, mark as inited. */
484 pool->cnt.parent = &ppool->cnt;
485 if (ppool && !pool->parent) {
486 pool->parent = ppool;
487 dmemcg_pool_get(ppool);
488 }
489 pool->inited = true;
490
491 pool = ppool;
492 }
493
494 return retpool;
495}
496
497static void dmemcg_free_rcu(struct rcu_head *rcu)
498{
499 struct dmem_cgroup_region *region = container_of(rcu, typeof(*region), rcu);
500 struct dmem_cgroup_pool_state *pool, *next;
501
502 list_for_each_entry_safe(pool, next, &region->pools, region_node)
503 free_cg_pool(pool);
504 kfree(region->name);
505 kfree(region);
506}
507
508static void dmemcg_free_region(struct kref *ref)
509{
510 struct dmem_cgroup_region *cgregion = container_of(ref, typeof(*cgregion), ref);
511
512 call_rcu(&cgregion->rcu, dmemcg_free_rcu);
513}
514
515/**
516 * dmem_cgroup_unregister_region() - Unregister a previously registered region.
517 * @region: The region to unregister.
518 *
519 * This function undoes dmem_cgroup_register_region. It drains any
520 * in-flight reclaim callbacks before returning, so the caller may safely
521 * free the resources pointed to by the @reclaim_priv that was passed at
522 * registration time.
523 */
524void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region)
525{
526 struct dmem_cgroup_pool_state *pool, *next;
527
528 if (!region)
529 return;
530
531 spin_lock(&dmemcg_lock);
532
533 /* Remove from global region list */
534 list_del_rcu(&region->region_node);
535
536 list_for_each_entry_safe(pool, next, &region->pools, region_node) {
537 list_del_rcu(&pool->css_node);
538 list_del(&pool->region_node);
539 dmemcg_pool_put(pool);
540 }
541
542 /*
543 * Ensure any RCU based lookups fail. Additionally,
544 * no new pools should be added to the dead region
545 * by get_cg_pool_unlocked.
546 */
547 WRITE_ONCE(region->unregistered, true);
548 spin_unlock(&dmemcg_lock);
549
550 synchronize_srcu(&dmemcg_srcu);
551
552 kref_put(&region->ref, dmemcg_free_region);
553}
554EXPORT_SYMBOL_GPL(dmem_cgroup_unregister_region);
555
556/**
557 * dmem_cgroup_register_region() - Register a regions for dev cgroup.
558 * @init: Initialization parameters for the region.
559 * @fmt: Region parameters to register
560 *
561 * This function registers a node in the dmem cgroup with the
562 * name given. After calling this function, the region can be
563 * used for allocations.
564 *
565 * Return: NULL or a struct on success, PTR_ERR on failure.
566 */
567struct dmem_cgroup_region *
568dmem_cgroup_register_region(const struct dmem_cgroup_init *init,
569 const char *fmt, ...)
570{
571 struct dmem_cgroup_region *ret;
572 char *region_name;
573 va_list ap;
574
575 if (!init || !init->size)
576 return NULL;
577
578 va_start(ap, fmt);
579 region_name = kvasprintf(GFP_KERNEL, fmt, ap);
580 va_end(ap);
581 if (!region_name)
582 return ERR_PTR(-ENOMEM);
583
584 ret = kzalloc_obj(*ret);
585 if (!ret) {
586 kfree(region_name);
587 return ERR_PTR(-ENOMEM);
588 }
589
590 INIT_LIST_HEAD(&ret->pools);
591 ret->name = region_name;
592 ret->size = init->size;
593 ret->ops = init->ops;
594 ret->reclaim_priv = init->reclaim_priv;
595 kref_init(&ret->ref);
596
597 spin_lock(&dmemcg_lock);
598 list_add_tail_rcu(&ret->region_node, &dmem_cgroup_regions);
599 spin_unlock(&dmemcg_lock);
600
601 return ret;
602}
603EXPORT_SYMBOL_GPL(dmem_cgroup_register_region);
604
605static struct dmem_cgroup_region *dmemcg_get_region_by_name(const char *name)
606{
607 struct dmem_cgroup_region *region;
608
609 list_for_each_entry_rcu(region, &dmem_cgroup_regions, region_node, spin_is_locked(&dmemcg_lock))
610 if (!strcmp(name, region->name) &&
611 kref_get_unless_zero(&region->ref))
612 return region;
613
614 return NULL;
615}
616
617/**
618 * dmem_cgroup_pool_state_put() - Drop a reference to a dmem_cgroup_pool_state
619 * @pool: &dmem_cgroup_pool_state
620 *
621 * Called to drop a reference to the limiting pool returned by
622 * dmem_cgroup_try_charge().
623 */
624void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool)
625{
626 if (pool) {
627 css_put(&pool->cs->css);
628 dmemcg_pool_put(pool);
629 }
630}
631EXPORT_SYMBOL_GPL(dmem_cgroup_pool_state_put);
632
633static struct dmem_cgroup_pool_state *
634get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region)
635{
636 struct dmem_cgroup_pool_state *pool, *allocpool = NULL;
637
638 /* fastpath lookup? */
639 rcu_read_lock();
640 pool = find_cg_pool_locked(cg, region);
641 if (pool && !READ_ONCE(pool->inited))
642 pool = NULL;
643 if (pool && !dmemcg_pool_tryget(pool))
644 pool = NULL;
645 rcu_read_unlock();
646
647 while (!pool) {
648 spin_lock(&dmemcg_lock);
649 if (!region->unregistered)
650 pool = get_cg_pool_locked(cg, region, &allocpool);
651 else
652 pool = ERR_PTR(-ENODEV);
653 if (!IS_ERR(pool))
654 dmemcg_pool_get(pool);
655 spin_unlock(&dmemcg_lock);
656
657 if (pool == ERR_PTR(-ENOMEM)) {
658 pool = NULL;
659 if (WARN_ON(allocpool))
660 continue;
661
662 allocpool = kzalloc_obj(*allocpool);
663 if (allocpool) {
664 pool = NULL;
665 continue;
666 }
667 pool = ERR_PTR(-ENOMEM);
668 }
669 }
670
671 kfree(allocpool);
672 return pool;
673}
674
675/**
676 * dmem_cgroup_uncharge() - Uncharge a pool.
677 * @pool: Pool to uncharge.
678 * @size: Size to uncharge.
679 *
680 * Undoes the effects of dmem_cgroup_try_charge.
681 * Must be called with the returned pool as argument,
682 * and same @index and @size.
683 */
684void dmem_cgroup_uncharge(struct dmem_cgroup_pool_state *pool, u64 size)
685{
686 if (!pool)
687 return;
688
689 page_counter_uncharge(&pool->cnt, size);
690 css_put(&pool->cs->css);
691 dmemcg_pool_put(pool);
692}
693EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge);
694
695/**
696 * dmem_cgroup_try_charge() - Try charging a new allocation to a region.
697 * @region: dmem region to charge
698 * @size: Size (in bytes) to charge.
699 * @ret_pool: On succesfull allocation, the pool that is charged.
700 * @ret_limit_pool: On a failed allocation, the limiting pool.
701 *
702 * This function charges the @region region for a size of @size bytes.
703 *
704 * If the function succeeds, @ret_pool is set, which must be passed to
705 * dmem_cgroup_uncharge() when undoing the allocation.
706 *
707 * When this function fails with -EAGAIN and @ret_limit_pool is non-null, it
708 * will be set to the pool for which the limit is hit. This can be used for
709 * eviction as argument to dmem_cgroup_evict_valuable(). This reference must be freed
710 * with @dmem_cgroup_pool_state_put().
711 *
712 * Return: 0 on success, -EAGAIN on hitting a limit, or a negative errno on failure.
713 */
714int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size,
715 struct dmem_cgroup_pool_state **ret_pool,
716 struct dmem_cgroup_pool_state **ret_limit_pool)
717{
718 struct dmemcg_state *cg;
719 struct dmem_cgroup_pool_state *pool;
720 struct page_counter *fail;
721 int ret;
722
723 *ret_pool = NULL;
724 if (ret_limit_pool)
725 *ret_limit_pool = NULL;
726
727 /*
728 * hold on to css, as cgroup can be removed but resource
729 * accounting happens on css.
730 */
731 cg = get_current_dmemcs();
732
733 pool = get_cg_pool_unlocked(cg, region);
734 if (IS_ERR(pool)) {
735 ret = PTR_ERR(pool);
736 goto err;
737 }
738
739 if (!page_counter_try_charge(&pool->cnt, size, &fail)) {
740 if (ret_limit_pool) {
741 *ret_limit_pool = container_of(fail, struct dmem_cgroup_pool_state, cnt);
742 css_get(&(*ret_limit_pool)->cs->css);
743 dmemcg_pool_get(*ret_limit_pool);
744 }
745 dmemcg_pool_put(pool);
746 ret = -EAGAIN;
747 goto err;
748 }
749
750 /* On success, reference from get_current_dmemcs is transferred to *ret_pool */
751 *ret_pool = pool;
752 return 0;
753
754err:
755 css_put(&cg->css);
756 return ret;
757}
758EXPORT_SYMBOL_GPL(dmem_cgroup_try_charge);
759
760/**
761 * dmem_cgroup_below_min() - Tests whether current usage is within min limit.
762 *
763 * @root: Root of the subtree to calculate protection for, or NULL to calculate global protection.
764 * @test: The pool to test the usage/min limit of.
765 *
766 * Return: true if usage is below min and the cgroup is protected, false otherwise.
767 */
768bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root,
769 struct dmem_cgroup_pool_state *test)
770{
771 if (root == test || !pool_parent(test))
772 return false;
773
774 if (!root) {
775 for (root = test; pool_parent(root); root = pool_parent(root))
776 {}
777 }
778
779 /*
780 * In mem_cgroup_below_min(), the memcg pendant, this call is missing.
781 * mem_cgroup_below_min() gets called during traversal of the cgroup tree, where
782 * protection is already calculated as part of the traversal. dmem cgroup eviction
783 * does not traverse the cgroup tree, so we need to recalculate effective protection
784 * here.
785 */
786 dmem_cgroup_calculate_protection(root, test);
787 return page_counter_read(&test->cnt) <= READ_ONCE(test->cnt.emin);
788}
789EXPORT_SYMBOL_GPL(dmem_cgroup_below_min);
790
791/**
792 * dmem_cgroup_below_low() - Tests whether current usage is within low limit.
793 *
794 * @root: Root of the subtree to calculate protection for, or NULL to calculate global protection.
795 * @test: The pool to test the usage/low limit of.
796 *
797 * Return: true if usage is below low and the cgroup is protected, false otherwise.
798 */
799bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root,
800 struct dmem_cgroup_pool_state *test)
801{
802 if (root == test || !pool_parent(test))
803 return false;
804
805 if (!root) {
806 for (root = test; pool_parent(root); root = pool_parent(root))
807 {}
808 }
809
810 /*
811 * In mem_cgroup_below_low(), the memcg pendant, this call is missing.
812 * mem_cgroup_below_low() gets called during traversal of the cgroup tree, where
813 * protection is already calculated as part of the traversal. dmem cgroup eviction
814 * does not traverse the cgroup tree, so we need to recalculate effective protection
815 * here.
816 */
817 dmem_cgroup_calculate_protection(root, test);
818 return page_counter_read(&test->cnt) <= READ_ONCE(test->cnt.elow);
819}
820EXPORT_SYMBOL_GPL(dmem_cgroup_below_low);
821
822/**
823 * dmem_cgroup_get_common_ancestor(): Find the first common ancestor of two pools.
824 * @a: First pool to find the common ancestor of.
825 * @b: First pool to find the common ancestor of.
826 *
827 * Return: The first pool that is a parent of both @a and @b, or NULL if either @a or @b are NULL,
828 * or if such a pool does not exist. A reference to the returned pool is grabbed and must be
829 * released by the caller when it is done using the pool.
830 */
831struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a,
832 struct dmem_cgroup_pool_state *b)
833{
834 struct cgroup *ancestor_cgroup;
835 struct cgroup_subsys_state *ancestor_css;
836 struct dmemcg_state *ancestor_dmemcs = NULL;
837 struct dmem_cgroup_pool_state *pool = NULL;
838
839 if (!a || !b)
840 return NULL;
841
842 ancestor_cgroup = cgroup_common_ancestor(a->cs->css.cgroup, b->cs->css.cgroup);
843 if (!ancestor_cgroup)
844 return NULL;
845
846 rcu_read_lock();
847 ancestor_css = cgroup_e_css(ancestor_cgroup, &dmem_cgrp_subsys);
848 if (css_tryget(ancestor_css))
849 ancestor_dmemcs = css_to_dmemcs(ancestor_css);
850 rcu_read_unlock();
851
852 if (ancestor_dmemcs) {
853 pool = get_cg_pool_unlocked(css_to_dmemcs(ancestor_css),
854 a->region);
855 if (WARN_ON(IS_ERR(pool))) {
856 pool = NULL;
857 css_put(ancestor_css);
858 }
859 }
860 return pool;
861}
862EXPORT_SYMBOL_GPL(dmem_cgroup_get_common_ancestor);
863
864static int dmem_cgroup_region_capacity_show(struct seq_file *sf, void *v)
865{
866 struct dmem_cgroup_region *region;
867
868 rcu_read_lock();
869 list_for_each_entry_rcu(region, &dmem_cgroup_regions, region_node) {
870 seq_puts(sf, region->name);
871 seq_printf(sf, " %llu\n", region->size);
872 }
873 rcu_read_unlock();
874 return 0;
875}
876
877static int dmemcg_parse_limit(char *options, u64 *new_limit)
878{
879 char *end;
880
881 if (!strcmp(options, "max")) {
882 *new_limit = PAGE_COUNTER_MAX;
883 return 0;
884 }
885
886 *new_limit = memparse(options, &end);
887 if (*end != '\0')
888 return -EINVAL;
889
890 return 0;
891}
892
893static ssize_t dmemcg_limit_write(struct kernfs_open_file *of,
894 char *buf, size_t nbytes, loff_t off,
895 void (*apply)(struct dmem_cgroup_pool_state *, u64, bool))
896{
897 struct dmemcg_state *dmemcs = css_to_dmemcs(of_css(of));
898 struct dmem_cgroup_pool_state *pool;
899 struct dmem_cgroup_region *region;
900 bool nonblock = of->file->f_flags & O_NONBLOCK;
901 char *region_name;
902 u64 new_limit;
903 int err;
904
905 buf = strstrip(buf);
906 region_name = strsep(&buf, " \t");
907 if (!buf || !region_name[0])
908 return -EINVAL;
909
910 rcu_read_lock();
911 region = dmemcg_get_region_by_name(region_name);
912 rcu_read_unlock();
913 if (!region)
914 return -EINVAL;
915
916 err = dmemcg_parse_limit(buf, &new_limit);
917 if (err < 0)
918 goto out_put;
919
920 pool = get_cg_pool_unlocked(dmemcs, region);
921 if (IS_ERR(pool)) {
922 err = PTR_ERR(pool);
923 goto out_put;
924 }
925
926 apply(pool, new_limit, nonblock);
927 dmemcg_pool_put(pool);
928
929out_put:
930 kref_put(&region->ref, dmemcg_free_region);
931
932 return err ?: nbytes;
933}
934
935static int dmemcg_limit_show(struct seq_file *sf, void *v,
936 u64 (*fn)(struct dmem_cgroup_pool_state *))
937{
938 struct dmemcg_state *dmemcs = css_to_dmemcs(seq_css(sf));
939 struct dmem_cgroup_region *region;
940
941 rcu_read_lock();
942 list_for_each_entry_rcu(region, &dmem_cgroup_regions, region_node) {
943 struct dmem_cgroup_pool_state *pool = find_cg_pool_locked(dmemcs, region);
944 u64 val;
945
946 seq_puts(sf, region->name);
947
948 val = fn(pool);
949 if (val < PAGE_COUNTER_MAX)
950 seq_printf(sf, " %lld\n", val);
951 else
952 seq_puts(sf, " max\n");
953 }
954 rcu_read_unlock();
955
956 return 0;
957}
958
959static int dmem_cgroup_region_peak_show(struct seq_file *sf, void *v)
960{
961 return dmemcg_limit_show(sf, v, get_resource_peak);
962}
963
964static int dmem_cgroup_region_current_show(struct seq_file *sf, void *v)
965{
966 return dmemcg_limit_show(sf, v, get_resource_current);
967}
968
969static int dmem_cgroup_region_min_show(struct seq_file *sf, void *v)
970{
971 return dmemcg_limit_show(sf, v, get_resource_min);
972}
973
974static ssize_t dmem_cgroup_region_min_write(struct kernfs_open_file *of,
975 char *buf, size_t nbytes, loff_t off)
976{
977 return dmemcg_limit_write(of, buf, nbytes, off, set_resource_min);
978}
979
980static int dmem_cgroup_region_low_show(struct seq_file *sf, void *v)
981{
982 return dmemcg_limit_show(sf, v, get_resource_low);
983}
984
985static ssize_t dmem_cgroup_region_low_write(struct kernfs_open_file *of,
986 char *buf, size_t nbytes, loff_t off)
987{
988 return dmemcg_limit_write(of, buf, nbytes, off, set_resource_low);
989}
990
991static int dmem_cgroup_region_max_show(struct seq_file *sf, void *v)
992{
993 return dmemcg_limit_show(sf, v, get_resource_max);
994}
995
996static ssize_t dmem_cgroup_region_max_write(struct kernfs_open_file *of,
997 char *buf, size_t nbytes, loff_t off)
998{
999 return dmemcg_limit_write(of, buf, nbytes, off, set_resource_max);
1000}
1001
1002static struct cftype files[] = {
1003 {
1004 .name = "capacity",
1005 .seq_show = dmem_cgroup_region_capacity_show,
1006 .flags = CFTYPE_ONLY_ON_ROOT,
1007 },
1008 {
1009 .name = "current",
1010 .seq_show = dmem_cgroup_region_current_show,
1011 },
1012 {
1013 .name = "peak",
1014 .seq_show = dmem_cgroup_region_peak_show,
1015 .flags = CFTYPE_NOT_ON_ROOT,
1016 },
1017 {
1018 .name = "min",
1019 .write = dmem_cgroup_region_min_write,
1020 .seq_show = dmem_cgroup_region_min_show,
1021 .flags = CFTYPE_NOT_ON_ROOT,
1022 },
1023 {
1024 .name = "low",
1025 .write = dmem_cgroup_region_low_write,
1026 .seq_show = dmem_cgroup_region_low_show,
1027 .flags = CFTYPE_NOT_ON_ROOT,
1028 },
1029 {
1030 .name = "max",
1031 .write = dmem_cgroup_region_max_write,
1032 .seq_show = dmem_cgroup_region_max_show,
1033 .flags = CFTYPE_NOT_ON_ROOT,
1034 },
1035 { } /* Zero entry terminates. */
1036};
1037
1038struct cgroup_subsys dmem_cgrp_subsys = {
1039 .css_alloc = dmemcs_alloc,
1040 .css_free = dmemcs_free,
1041 .css_offline = dmemcs_offline,
1042 .legacy_cftypes = files,
1043 .dfl_cftypes = files,
1044};