projects/linux·blob master · refs

6218 lines · 162.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/* memcontrol.c - Memory Controller
3 *
4 * Copyright IBM Corporation, 2007
5 * Author Balbir Singh <[email protected]>
6 *
7 * Copyright 2007 OpenVZ SWsoft Inc
8 * Author: Pavel Emelianov <[email protected]>
9 *
10 * Memory thresholds
11 * Copyright (C) 2009 Nokia Corporation
12 * Author: Kirill A. Shutemov
13 *
14 * Kernel Memory Controller
15 * Copyright (C) 2012 Parallels Inc. and Google Inc.
16 * Authors: Glauber Costa and Suleiman Souhlal
17 *
18 * Native page reclaim
19 * Charge lifetime sanitation
20 * Lockless page tracking & accounting
21 * Unified hierarchy configuration model
22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
23 *
24 * Per memcg lru locking
25 * Copyright (C) 2020 Alibaba, Inc, Alex Shi
26 */
27
28#include <linux/cgroup-defs.h>
29#include <linux/page_counter.h>
30#include <linux/memcontrol.h>
31#include <linux/cgroup.h>
32#include <linux/cpuset.h>
33#include <linux/sched/mm.h>
34#include <linux/shmem_fs.h>
35#include <linux/hugetlb.h>
36#include <linux/pagemap.h>
37#include <linux/folio_batch.h>
38#include <linux/vm_event_item.h>
39#include <linux/smp.h>
40#include <linux/page-flags.h>
41#include <linux/backing-dev.h>
42#include <linux/bit_spinlock.h>
43#include <linux/rcupdate.h>
44#include <linux/limits.h>
45#include <linux/export.h>
46#include <linux/list.h>
47#include <linux/mutex.h>
48#include <linux/rbtree.h>
49#include <linux/slab.h>
50#include <linux/swapops.h>
51#include <linux/spinlock.h>
52#include <linux/fs.h>
53#include <linux/seq_file.h>
54#include <linux/vmpressure.h>
55#include <linux/memremap.h>
56#include <linux/mm_inline.h>
57#include <linux/cpu.h>
58#include <linux/oom.h>
59#include <linux/lockdep.h>
60#include <linux/resume_user_mode.h>
61#include <linux/psi.h>
62#include <linux/seq_buf.h>
63#include <linux/sched/isolation.h>
64#include <linux/kmemleak.h>
65#include "internal.h"
66#include "swap.h"
67#include "swap_table.h"
68#include <net/sock.h>
69#include <net/ip.h>
70#include "slab.h"
71#include "memcontrol-v1.h"
72
73#include <linux/uaccess.h>
74
75#define CREATE_TRACE_POINTS
76#include <trace/events/memcg.h>
77#undef CREATE_TRACE_POINTS
78
79#include <trace/events/vmscan.h>
80
81struct cgroup_subsys memory_cgrp_subsys __read_mostly;
82EXPORT_SYMBOL(memory_cgrp_subsys);
83
84struct mem_cgroup *root_mem_cgroup __read_mostly;
85EXPORT_SYMBOL(root_mem_cgroup);
86
87/* Active memory cgroup to use from an interrupt context */
88DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
89EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
90
91/* Socket memory accounting disabled? */
92static bool cgroup_memory_nosocket __ro_after_init;
93
94/* Kernel memory accounting disabled? */
95static bool cgroup_memory_nokmem __ro_after_init;
96
97/* BPF memory accounting disabled? */
98static bool cgroup_memory_nobpf __ro_after_init;
99
100static struct workqueue_struct *memcg_wq __ro_after_init;
101
102static struct kmem_cache *memcg_cachep;
103static struct kmem_cache *memcg_pn_cachep;
104
105#ifdef CONFIG_CGROUP_WRITEBACK
106static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
107#endif
108
109static inline bool task_is_dying(void)
110{
111 return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
112 (current->flags & PF_EXITING);
113}
114
115/* Some nice accessors for the vmpressure. */
116struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
117{
118 if (!memcg)
119 memcg = root_mem_cgroup;
120 return &memcg->vmpressure;
121}
122
123struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr)
124{
125 return container_of(vmpr, struct mem_cgroup, vmpressure);
126}
127
128#define SEQ_BUF_SIZE SZ_4K
129#define CURRENT_OBJCG_UPDATE_BIT 0
130#define CURRENT_OBJCG_UPDATE_FLAG (1UL << CURRENT_OBJCG_UPDATE_BIT)
131
132static DEFINE_SPINLOCK(objcg_lock);
133
134bool mem_cgroup_kmem_disabled(void)
135{
136 return cgroup_memory_nokmem;
137}
138
139static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages);
140
141static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages)
142{
143 mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);
144 memcg1_account_kmem(memcg, -nr_pages);
145 if (!mem_cgroup_is_root(memcg))
146 memcg_uncharge(memcg, nr_pages);
147}
148
149static void obj_cgroup_release(struct percpu_ref *ref)
150{
151 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
152 unsigned int nr_bytes;
153 unsigned int nr_pages;
154 unsigned long flags;
155
156 /*
157 * At this point all allocated objects are freed, and
158 * objcg->nr_charged_bytes can't have an arbitrary byte value.
159 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
160 *
161 * The following sequence can lead to it:
162 * 1) CPU0: objcg cached in one of stock->cached[i]
163 * 2) CPU1: we do a small allocation (e.g. 92 bytes),
164 * PAGE_SIZE bytes are charged
165 * 3) CPU1: a process from another memcg is allocating something,
166 * the stock if flushed,
167 * objcg->nr_charged_bytes = PAGE_SIZE - 92
168 * 4) CPU0: we do release this object,
169 * 92 bytes are added to stock->nr_bytes[i]
170 * 5) CPU0: stock is flushed,
171 * 92 bytes are added to objcg->nr_charged_bytes
172 *
173 * In the result, nr_charged_bytes == PAGE_SIZE.
174 * This page will be uncharged in obj_cgroup_release().
175 */
176 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
177 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
178 nr_pages = nr_bytes >> PAGE_SHIFT;
179
180 if (nr_pages) {
181 struct mem_cgroup *memcg;
182
183 memcg = get_mem_cgroup_from_objcg(objcg);
184 memcg_uncharge_kmem(memcg, nr_pages);
185 mem_cgroup_put(memcg);
186 }
187
188 spin_lock_irqsave(&objcg_lock, flags);
189 list_del(&objcg->list);
190 spin_unlock_irqrestore(&objcg_lock, flags);
191
192 percpu_ref_exit(ref);
193 kfree_rcu(objcg, rcu);
194}
195
196static struct obj_cgroup *obj_cgroup_alloc(void)
197{
198 struct obj_cgroup *objcg;
199 int ret;
200
201 objcg = kzalloc_obj(struct obj_cgroup);
202 if (!objcg)
203 return NULL;
204
205 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
206 GFP_KERNEL);
207 if (ret) {
208 kfree(objcg);
209 return NULL;
210 }
211 INIT_LIST_HEAD(&objcg->list);
212 return objcg;
213}
214
215static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memcg,
216 struct mem_cgroup *parent,
217 int nid)
218{
219 struct obj_cgroup *objcg, *iter;
220 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
221 struct mem_cgroup_per_node *parent_pn = parent->nodeinfo[nid];
222
223 objcg = rcu_replace_pointer(pn->objcg, NULL, true);
224 /* 1) Ready to reparent active objcg. */
225 list_add(&objcg->list, &pn->objcg_list);
226 /* 2) Reparent active objcg and already reparented objcgs to parent. */
227 list_for_each_entry(iter, &pn->objcg_list, list)
228 WRITE_ONCE(iter->memcg, parent);
229 /* 3) Move already reparented objcgs to the parent's list */
230 list_splice(&pn->objcg_list, &parent_pn->objcg_list);
231
232 return objcg;
233}
234
235#ifdef CONFIG_MEMCG_V1
236static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force);
237
238static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
239{
240 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
241 return;
242
243 /*
244 * Reparent stats exposed non-hierarchically. Flush @memcg's stats first
245 * to read its stats accurately , and conservatively flush @parent's
246 * stats after reparenting to avoid hiding a potentially large stat
247 * update (e.g. from callers of mem_cgroup_flush_stats_ratelimited()).
248 */
249 __mem_cgroup_flush_stats(memcg, true);
250
251 /* The following counts are all non-hierarchical and need to be reparented. */
252 reparent_memcg1_state_local(memcg, parent);
253 reparent_memcg1_lruvec_state_local(memcg, parent);
254
255 __mem_cgroup_flush_stats(parent, true);
256}
257#else
258static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
259{
260}
261#endif
262
263static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
264{
265 spin_lock_irq(&objcg_lock);
266 spin_lock_nested(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock, 1);
267 spin_lock_nested(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock, 2);
268}
269
270static inline void reparent_unlocks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
271{
272 spin_unlock(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock);
273 spin_unlock(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock);
274 spin_unlock_irq(&objcg_lock);
275}
276
277static void memcg_reparent_objcgs(struct mem_cgroup *memcg)
278{
279 struct obj_cgroup *objcg;
280 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
281 int nid;
282
283 for_each_node(nid) {
284retry:
285 if (lru_gen_enabled())
286 max_lru_gen_memcg(parent, nid);
287
288 reparent_locks(memcg, parent, nid);
289
290 if (lru_gen_enabled()) {
291 if (!recheck_lru_gen_max_memcg(parent, nid)) {
292 reparent_unlocks(memcg, parent, nid);
293 cond_resched();
294 goto retry;
295 }
296 lru_gen_reparent_memcg(memcg, parent, nid);
297 } else {
298 lru_reparent_memcg(memcg, parent, nid);
299 }
300
301 objcg = __memcg_reparent_objcgs(memcg, parent, nid);
302
303 reparent_unlocks(memcg, parent, nid);
304
305 percpu_ref_kill(&objcg->refcnt);
306 }
307
308 reparent_state_local(memcg, parent);
309}
310
311/*
312 * A lot of the calls to the cache allocation functions are expected to be
313 * inlined by the compiler. Since the calls to memcg_slab_post_alloc_hook() are
314 * conditional to this static branch, we'll have to allow modules that does
315 * kmem_cache_alloc and the such to see this symbol as well
316 */
317DEFINE_STATIC_KEY_FALSE(memcg_kmem_online_key);
318EXPORT_SYMBOL(memcg_kmem_online_key);
319
320DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key);
321EXPORT_SYMBOL(memcg_bpf_enabled_key);
322
323/**
324 * get_mem_cgroup_css_from_folio - acquire a css of the memcg associated with a folio
325 * @folio: folio of interest
326 *
327 * If memcg is bound to the default hierarchy, css of the memcg associated
328 * with @folio is returned. The returned css remains associated with @folio
329 * until it is released.
330 *
331 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
332 * is returned.
333 */
334struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio)
335{
336 struct mem_cgroup *memcg;
337
338 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
339 return &root_mem_cgroup->css;
340
341 memcg = get_mem_cgroup_from_folio(folio);
342
343 return memcg ? &memcg->css : &root_mem_cgroup->css;
344}
345
346/**
347 * page_cgroup_ino - return inode number of the memcg a page is charged to
348 * @page: the page
349 *
350 * Look up the closest online ancestor of the memory cgroup @page is charged to
351 * and return its inode number or 0 if @page is not charged to any cgroup. It
352 * is safe to call this function without holding a reference to @page.
353 *
354 * Note, this function is inherently racy, because there is nothing to prevent
355 * the cgroup inode from getting torn down and potentially reallocated a moment
356 * after page_cgroup_ino() returns, so it only should be used by callers that
357 * do not care (such as procfs interfaces).
358 */
359ino_t page_cgroup_ino(struct page *page)
360{
361 struct mem_cgroup *memcg;
362 unsigned long ino = 0;
363
364 rcu_read_lock();
365 /* page_folio() is racy here, but the entire function is racy anyway */
366 memcg = folio_memcg_check(page_folio(page));
367
368 while (memcg && !css_is_online(&memcg->css))
369 memcg = parent_mem_cgroup(memcg);
370 if (memcg)
371 ino = cgroup_ino(memcg->css.cgroup);
372 rcu_read_unlock();
373 return ino;
374}
375EXPORT_SYMBOL_GPL(page_cgroup_ino);
376
377/* Subset of node_stat_item for memcg stats */
378static const unsigned int memcg_node_stat_items[] = {
379 NR_INACTIVE_ANON,
380 NR_ACTIVE_ANON,
381 NR_INACTIVE_FILE,
382 NR_ACTIVE_FILE,
383 NR_UNEVICTABLE,
384 NR_SLAB_RECLAIMABLE_B,
385 NR_SLAB_UNRECLAIMABLE_B,
386 WORKINGSET_REFAULT_ANON,
387 WORKINGSET_REFAULT_FILE,
388 WORKINGSET_ACTIVATE_ANON,
389 WORKINGSET_ACTIVATE_FILE,
390 WORKINGSET_RESTORE_ANON,
391 WORKINGSET_RESTORE_FILE,
392 WORKINGSET_NODERECLAIM,
393 NR_ANON_MAPPED,
394 NR_FILE_MAPPED,
395 NR_FILE_PAGES,
396 NR_FILE_DIRTY,
397 NR_WRITEBACK,
398 NR_SHMEM,
399 NR_SHMEM_THPS,
400 NR_FILE_THPS,
401 NR_ANON_THPS,
402 NR_VMSCAN_WRITE,
403 NR_VMALLOC,
404 NR_KERNEL_STACK_KB,
405 NR_PAGETABLE,
406 NR_SECONDARY_PAGETABLE,
407#ifdef CONFIG_SWAP
408 NR_SWAPCACHE,
409#endif
410#ifdef CONFIG_NUMA_BALANCING
411 PGPROMOTE_SUCCESS,
412#endif
413 PGDEMOTE_KSWAPD,
414 PGDEMOTE_DIRECT,
415 PGDEMOTE_KHUGEPAGED,
416 PGDEMOTE_PROACTIVE,
417 PGSTEAL_KSWAPD,
418 PGSTEAL_DIRECT,
419 PGSTEAL_KHUGEPAGED,
420 PGSTEAL_PROACTIVE,
421 PGSTEAL_ANON,
422 PGSTEAL_FILE,
423 PGSCAN_KSWAPD,
424 PGSCAN_DIRECT,
425 PGSCAN_KHUGEPAGED,
426 PGSCAN_PROACTIVE,
427 PGSCAN_ANON,
428 PGSCAN_FILE,
429 PGROTATE_ANON,
430 PGROTATE_FILE,
431 PGREFILL,
432#ifdef CONFIG_HUGETLB_PAGE
433 NR_HUGETLB,
434#endif
435};
436
437static const unsigned int memcg_stat_items[] = {
438 MEMCG_SWAP,
439 MEMCG_SOCK,
440 MEMCG_PERCPU_B,
441 MEMCG_KMEM,
442 MEMCG_ZSWAP_B,
443 MEMCG_ZSWAPPED,
444 MEMCG_ZSWAP_INCOMP,
445};
446
447#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items)
448#define MEMCG_VMSTAT_SIZE (NR_MEMCG_NODE_STAT_ITEMS + \
449 ARRAY_SIZE(memcg_stat_items))
450#define BAD_STAT_IDX(index) ((u32)(index) >= U8_MAX)
451static u8 mem_cgroup_stats_index[MEMCG_NR_STAT] __read_mostly;
452
453static void init_memcg_stats(void)
454{
455 u8 i, j = 0;
456
457 BUILD_BUG_ON(MEMCG_NR_STAT >= U8_MAX);
458
459 memset(mem_cgroup_stats_index, U8_MAX, sizeof(mem_cgroup_stats_index));
460
461 for (i = 0; i < NR_MEMCG_NODE_STAT_ITEMS; ++i, ++j)
462 mem_cgroup_stats_index[memcg_node_stat_items[i]] = j;
463
464 for (i = 0; i < ARRAY_SIZE(memcg_stat_items); ++i, ++j)
465 mem_cgroup_stats_index[memcg_stat_items[i]] = j;
466}
467
468static inline int memcg_stats_index(int idx)
469{
470 return mem_cgroup_stats_index[idx];
471}
472
473struct lruvec_stats_percpu {
474 /* Local (CPU and cgroup) state */
475 long state[NR_MEMCG_NODE_STAT_ITEMS];
476
477 /* Delta calculation for lockless upward propagation */
478 long state_prev[NR_MEMCG_NODE_STAT_ITEMS];
479};
480
481struct lruvec_stats {
482 /* Aggregated (CPU and subtree) state */
483 long state[NR_MEMCG_NODE_STAT_ITEMS];
484
485 /* Non-hierarchical (CPU aggregated) state */
486 long state_local[NR_MEMCG_NODE_STAT_ITEMS];
487
488 /* Pending child counts during tree propagation */
489 long state_pending[NR_MEMCG_NODE_STAT_ITEMS];
490};
491
492unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx)
493{
494 struct mem_cgroup_per_node *pn;
495 long x;
496 int i;
497
498 if (mem_cgroup_disabled())
499 return node_page_state(lruvec_pgdat(lruvec), idx);
500
501 i = memcg_stats_index(idx);
502 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
503 return 0;
504
505 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
506 x = READ_ONCE(pn->lruvec_stats->state[i]);
507#ifdef CONFIG_SMP
508 if (x < 0)
509 x = 0;
510#endif
511 return x;
512}
513
514/**
515 * lruvec_page_state_monotonic - non-clamping lruvec stat read for delta sampling
516 * @lruvec: the LRU vector to read from
517 * @idx: the node_stat_item to read
518 *
519 * Returns the raw state[idx] value cast to unsigned long, skipping the
520 * clamp-negative-to-zero step in lruvec_page_state(). Intended for callers
521 * that snapshot a monotonically-incremented counter and subtract two
522 * samples: unsigned modular arithmetic then yields the correct delta across
523 * a signed-long wraparound (a real hazard on 32-bit) that the clamp would
524 * otherwise turn into a huge spurious delta.
525 *
526 * Do NOT use for non-monotonic page-count reads where a transient negative
527 * reading from per-CPU delta skew must present as zero.
528 *
529 * XXX: This helper (and its node/global peers) exists because some
530 * monotonically-incremented event counters are stored in
531 * enum node_stat_item.
532 */
533unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
534 enum node_stat_item idx)
535{
536 struct mem_cgroup_per_node *pn;
537 int i;
538
539 if (mem_cgroup_disabled())
540 return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
541
542 i = memcg_stats_index(idx);
543 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
544 return 0;
545
546 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
547 return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]);
548}
549
550unsigned long lruvec_page_state_local(struct lruvec *lruvec,
551 enum node_stat_item idx)
552{
553 struct mem_cgroup_per_node *pn;
554 long x;
555 int i;
556
557 if (mem_cgroup_disabled())
558 return node_page_state(lruvec_pgdat(lruvec), idx);
559
560 i = memcg_stats_index(idx);
561 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
562 return 0;
563
564 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
565 x = READ_ONCE(pn->lruvec_stats->state_local[i]);
566#ifdef CONFIG_SMP
567 if (x < 0)
568 x = 0;
569#endif
570 return x;
571}
572
573#ifdef CONFIG_MEMCG_V1
574static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn,
575 enum node_stat_item idx, long val);
576
577void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg,
578 struct mem_cgroup *parent, int idx)
579{
580 int nid;
581
582 for_each_node(nid) {
583 struct lruvec *child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
584 struct lruvec *parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
585 unsigned long value = lruvec_page_state_local(child_lruvec, idx);
586 struct mem_cgroup_per_node *child_pn, *parent_pn;
587
588 child_pn = container_of(child_lruvec, struct mem_cgroup_per_node, lruvec);
589 parent_pn = container_of(parent_lruvec, struct mem_cgroup_per_node, lruvec);
590
591 __mod_memcg_lruvec_state(child_pn, idx, -value);
592 __mod_memcg_lruvec_state(parent_pn, idx, value);
593 }
594}
595#endif
596
597/* Subset of vm_event_item to report for memcg event stats */
598static const unsigned int memcg_vm_event_stat[] = {
599#ifdef CONFIG_MEMCG_V1
600 PGPGIN,
601 PGPGOUT,
602#endif
603 PSWPIN,
604 PSWPOUT,
605 PGFAULT,
606 PGMAJFAULT,
607 PGACTIVATE,
608 PGDEACTIVATE,
609 PGLAZYFREE,
610 PGLAZYFREED,
611#ifdef CONFIG_SWAP
612 SWPIN_ZERO,
613 SWPOUT_ZERO,
614#endif
615#ifdef CONFIG_ZSWAP
616 ZSWPIN,
617 ZSWPOUT,
618 ZSWPWB,
619#endif
620#ifdef CONFIG_TRANSPARENT_HUGEPAGE
621 THP_FAULT_ALLOC,
622 THP_COLLAPSE_ALLOC,
623 THP_SWPOUT,
624 THP_SWPOUT_FALLBACK,
625#endif
626#ifdef CONFIG_NUMA_BALANCING
627 NUMA_PAGE_MIGRATE,
628 NUMA_PTE_UPDATES,
629 NUMA_HINT_FAULTS,
630#endif
631};
632
633#define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat)
634static u8 mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly;
635
636static void init_memcg_events(void)
637{
638 u8 i;
639
640 BUILD_BUG_ON(NR_VM_EVENT_ITEMS >= U8_MAX);
641
642 memset(mem_cgroup_events_index, U8_MAX,
643 sizeof(mem_cgroup_events_index));
644
645 for (i = 0; i < NR_MEMCG_EVENTS; ++i)
646 mem_cgroup_events_index[memcg_vm_event_stat[i]] = i;
647}
648
649static inline int memcg_events_index(enum vm_event_item idx)
650{
651 return mem_cgroup_events_index[idx];
652}
653
654struct memcg_vmstats_percpu {
655 /* Stats updates since the last flush */
656 unsigned long stats_updates;
657
658 /* Cached pointers for fast iteration in memcg_rstat_updated() */
659 struct memcg_vmstats_percpu __percpu *parent_pcpu;
660 struct memcg_vmstats *vmstats;
661
662 /* The above should fit a single cacheline for memcg_rstat_updated() */
663
664 /* Local (CPU and cgroup) page state & events */
665 long state[MEMCG_VMSTAT_SIZE];
666 unsigned long events[NR_MEMCG_EVENTS];
667
668 /* Delta calculation for lockless upward propagation */
669 long state_prev[MEMCG_VMSTAT_SIZE];
670 unsigned long events_prev[NR_MEMCG_EVENTS];
671} ____cacheline_aligned;
672
673struct memcg_vmstats {
674 /* Aggregated (CPU and subtree) page state & events */
675 long state[MEMCG_VMSTAT_SIZE];
676 unsigned long events[NR_MEMCG_EVENTS];
677
678 /* Non-hierarchical (CPU aggregated) page state & events */
679 long state_local[MEMCG_VMSTAT_SIZE];
680 unsigned long events_local[NR_MEMCG_EVENTS];
681
682 /* Pending child counts during tree propagation */
683 long state_pending[MEMCG_VMSTAT_SIZE];
684 unsigned long events_pending[NR_MEMCG_EVENTS];
685
686 /* Stats updates since the last flush */
687 atomic_long_t stats_updates;
688};
689
690/*
691 * memcg and lruvec stats flushing
692 *
693 * Many codepaths leading to stats update or read are performance sensitive and
694 * adding stats flushing in such codepaths is not desirable. So, to optimize the
695 * flushing the kernel does:
696 *
697 * 1) Periodically and asynchronously flush the stats every 2 seconds to not let
698 * rstat update tree grow unbounded.
699 *
700 * 2) Flush the stats synchronously on reader side only when there are more than
701 * (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization
702 * will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but
703 * only for 2 seconds due to (1).
704 */
705static void flush_memcg_stats_dwork(struct work_struct *w);
706static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);
707static u64 flush_last_time;
708
709#define FLUSH_TIME (2UL*HZ)
710
711static bool memcg_vmstats_needs_flush(struct memcg_vmstats *vmstats)
712{
713 return atomic_long_read(&vmstats->stats_updates) >
714 MEMCG_CHARGE_BATCH * num_online_cpus();
715}
716
717static inline void memcg_rstat_updated(struct mem_cgroup *memcg, long val,
718 int cpu)
719{
720 struct memcg_vmstats_percpu __percpu *statc_pcpu;
721 struct memcg_vmstats_percpu *statc;
722 unsigned long stats_updates;
723
724 if (!val)
725 return;
726
727 __css_rstat_updated(&memcg->css, cpu);
728 statc_pcpu = memcg->vmstats_percpu;
729 for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
730 statc = this_cpu_ptr(statc_pcpu);
731 /*
732 * If @memcg is already flushable then all its ancestors are
733 * flushable as well and also there is no need to increase
734 * stats_updates.
735 */
736 if (memcg_vmstats_needs_flush(statc->vmstats))
737 break;
738
739 stats_updates = this_cpu_add_return(statc_pcpu->stats_updates,
740 abs(val));
741 if (stats_updates < MEMCG_CHARGE_BATCH)
742 continue;
743
744 stats_updates = this_cpu_xchg(statc_pcpu->stats_updates, 0);
745 atomic_long_add(stats_updates, &statc->vmstats->stats_updates);
746 }
747}
748
749static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force)
750{
751 bool needs_flush = memcg_vmstats_needs_flush(memcg->vmstats);
752
753 trace_memcg_flush_stats(memcg, atomic_long_read(&memcg->vmstats->stats_updates),
754 force, needs_flush);
755
756 if (!force && !needs_flush)
757 return;
758
759 if (mem_cgroup_is_root(memcg))
760 WRITE_ONCE(flush_last_time, jiffies_64);
761
762 css_rstat_flush(&memcg->css);
763}
764
765/*
766 * mem_cgroup_flush_stats - flush the stats of a memory cgroup subtree
767 * @memcg: root of the subtree to flush
768 *
769 * Flushing is serialized by the underlying global rstat lock. There is also a
770 * minimum amount of work to be done even if there are no stat updates to flush.
771 * Hence, we only flush the stats if the updates delta exceeds a threshold. This
772 * avoids unnecessary work and contention on the underlying lock.
773 */
774void mem_cgroup_flush_stats(struct mem_cgroup *memcg)
775{
776 if (mem_cgroup_disabled())
777 return;
778
779 if (!memcg)
780 memcg = root_mem_cgroup;
781
782 __mem_cgroup_flush_stats(memcg, false);
783}
784
785void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg)
786{
787 /* Only flush if the periodic flusher is one full cycle late */
788 if (time_after64(jiffies_64, READ_ONCE(flush_last_time) + 2*FLUSH_TIME))
789 mem_cgroup_flush_stats(memcg);
790}
791
792static void flush_memcg_stats_dwork(struct work_struct *w)
793{
794 /*
795 * Deliberately ignore memcg_vmstats_needs_flush() here so that flushing
796 * in latency-sensitive paths is as cheap as possible.
797 */
798 __mem_cgroup_flush_stats(root_mem_cgroup, true);
799 queue_delayed_work(system_dfl_wq, &stats_flush_dwork, FLUSH_TIME);
800}
801
802unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
803{
804 long x;
805 int i = memcg_stats_index(idx);
806
807 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
808 return 0;
809
810 x = READ_ONCE(memcg->vmstats->state[i]);
811#ifdef CONFIG_SMP
812 if (x < 0)
813 x = 0;
814#endif
815 return x;
816}
817
818bool memcg_stat_item_valid(int idx)
819{
820 if ((u32)idx >= MEMCG_NR_STAT)
821 return false;
822
823 return !BAD_STAT_IDX(memcg_stats_index(idx));
824}
825
826static int memcg_page_state_unit(int item);
827
828/*
829 * Normalize the value passed into memcg_rstat_updated() to be in pages. Round
830 * up non-zero sub-page updates to 1 page as zero page updates are ignored.
831 */
832static long memcg_state_val_in_pages(int idx, long val)
833{
834 int unit = memcg_page_state_unit(idx);
835 long res;
836
837 if (!val || unit == PAGE_SIZE)
838 return val;
839
840 /* Get the absolute value of (val * unit / PAGE_SIZE). */
841 res = mult_frac(abs(val), unit, PAGE_SIZE);
842 /* Round up zero values. */
843 res = res ? : 1;
844
845 return val < 0 ? -res : res;
846}
847
848#ifdef CONFIG_MEMCG_V1
849/*
850 * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race with
851 * reparenting of non-hierarchical state_locals.
852 */
853static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg,
854 bool *rcu_locked)
855{
856 /* Rebinding can cause this value to be changed at runtime */
857 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
858 *rcu_locked = false;
859 return memcg;
860 }
861
862 rcu_read_lock();
863 *rcu_locked = true;
864
865 while (memcg_is_dying(memcg))
866 memcg = parent_mem_cgroup(memcg);
867
868 return memcg;
869}
870
871static inline void get_non_dying_memcg_end(bool rcu_locked)
872{
873 if (!rcu_locked)
874 return;
875
876 rcu_read_unlock();
877}
878#else
879static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg,
880 bool *rcu_locked)
881{
882 return memcg;
883}
884
885static inline void get_non_dying_memcg_end(bool rcu_locked)
886{
887}
888#endif
889
890static void __mod_memcg_state(struct mem_cgroup *memcg,
891 enum memcg_stat_item idx, long val)
892{
893 int i = memcg_stats_index(idx);
894 int cpu;
895
896 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
897 return;
898
899 cpu = get_cpu();
900
901 this_cpu_add(memcg->vmstats_percpu->state[i], val);
902 val = memcg_state_val_in_pages(idx, val);
903 memcg_rstat_updated(memcg, val, cpu);
904
905 trace_mod_memcg_state(memcg, idx, val);
906
907 put_cpu();
908}
909
910/**
911 * mod_memcg_state - update cgroup memory statistics
912 * @memcg: the memory cgroup
913 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
914 * @val: delta to add to the counter, can be negative
915 */
916void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
917 int val)
918{
919 bool rcu_locked = false;
920
921 if (mem_cgroup_disabled())
922 return;
923
924 memcg = get_non_dying_memcg_start(memcg, &rcu_locked);
925 __mod_memcg_state(memcg, idx, val);
926 get_non_dying_memcg_end(rcu_locked);
927}
928
929#ifdef CONFIG_MEMCG_V1
930/* idx can be of type enum memcg_stat_item or node_stat_item. */
931unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
932{
933 long x;
934 int i = memcg_stats_index(idx);
935
936 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
937 return 0;
938
939 x = READ_ONCE(memcg->vmstats->state_local[i]);
940#ifdef CONFIG_SMP
941 if (x < 0)
942 x = 0;
943#endif
944 return x;
945}
946
947void reparent_memcg_state_local(struct mem_cgroup *memcg,
948 struct mem_cgroup *parent, int idx)
949{
950 unsigned long value = memcg_page_state_local(memcg, idx);
951
952 __mod_memcg_state(memcg, idx, -value);
953 __mod_memcg_state(parent, idx, value);
954}
955#endif
956
957static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn,
958 enum node_stat_item idx, long val)
959{
960 struct mem_cgroup *memcg = pn->memcg;
961 int i = memcg_stats_index(idx);
962 int cpu;
963
964 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
965 return;
966
967 cpu = get_cpu();
968
969 /* Update memcg */
970 this_cpu_add(memcg->vmstats_percpu->state[i], val);
971
972 /* Update lruvec */
973 this_cpu_add(pn->lruvec_stats_percpu->state[i], val);
974
975 val = memcg_state_val_in_pages(idx, val);
976 memcg_rstat_updated(memcg, val, cpu);
977 trace_mod_memcg_lruvec_state(memcg, idx, val);
978
979 put_cpu();
980}
981
982static void mod_memcg_lruvec_state(struct lruvec *lruvec,
983 enum node_stat_item idx,
984 int val)
985{
986 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
987 struct mem_cgroup_per_node *pn;
988 struct mem_cgroup *memcg;
989 bool rcu_locked = false;
990
991 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
992 memcg = get_non_dying_memcg_start(pn->memcg, &rcu_locked);
993 pn = memcg->nodeinfo[pgdat->node_id];
994
995 __mod_memcg_lruvec_state(pn, idx, val);
996
997 get_non_dying_memcg_end(rcu_locked);
998}
999
1000/**
1001 * mod_lruvec_state - update lruvec memory statistics
1002 * @lruvec: the lruvec
1003 * @idx: the stat item
1004 * @val: delta to add to the counter, can be negative
1005 *
1006 * The lruvec is the intersection of the NUMA node and a cgroup. This
1007 * function updates the all three counters that are affected by a
1008 * change of state at this level: per-node, per-cgroup, per-lruvec.
1009 */
1010void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
1011 int val)
1012{
1013 /* Update node */
1014 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
1015
1016 /* Update memcg and lruvec */
1017 if (!mem_cgroup_disabled())
1018 mod_memcg_lruvec_state(lruvec, idx, val);
1019}
1020
1021void lruvec_stat_mod_folio(struct folio *folio, enum node_stat_item idx,
1022 int val)
1023{
1024 struct mem_cgroup *memcg;
1025 pg_data_t *pgdat = folio_pgdat(folio);
1026 struct lruvec *lruvec;
1027
1028 rcu_read_lock();
1029 memcg = folio_memcg(folio);
1030 /* Untracked pages have no memcg, no lruvec. Update only the node */
1031 if (!memcg) {
1032 rcu_read_unlock();
1033 mod_node_page_state(pgdat, idx, val);
1034 return;
1035 }
1036
1037 lruvec = mem_cgroup_lruvec(memcg, pgdat);
1038 mod_lruvec_state(lruvec, idx, val);
1039 rcu_read_unlock();
1040}
1041EXPORT_SYMBOL(lruvec_stat_mod_folio);
1042
1043void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val)
1044{
1045 pg_data_t *pgdat = page_pgdat(virt_to_page(p));
1046 struct mem_cgroup *memcg;
1047 struct lruvec *lruvec;
1048
1049 rcu_read_lock();
1050 memcg = mem_cgroup_from_virt(p);
1051
1052 /*
1053 * Untracked pages have no memcg, no lruvec. Update only the
1054 * node. If we reparent the slab objects to the root memcg,
1055 * when we free the slab object, we need to update the per-memcg
1056 * vmstats to keep it correct for the root memcg.
1057 */
1058 if (!memcg) {
1059 mod_node_page_state(pgdat, idx, val);
1060 } else {
1061 lruvec = mem_cgroup_lruvec(memcg, pgdat);
1062 mod_lruvec_state(lruvec, idx, val);
1063 }
1064 rcu_read_unlock();
1065}
1066
1067/**
1068 * count_memcg_events - account VM events in a cgroup
1069 * @memcg: the memory cgroup
1070 * @idx: the event item
1071 * @count: the number of events that occurred
1072 */
1073void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
1074 unsigned long count)
1075{
1076 int i = memcg_events_index(idx);
1077 int cpu;
1078
1079 if (mem_cgroup_disabled())
1080 return;
1081
1082 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
1083 return;
1084
1085 cpu = get_cpu();
1086
1087 this_cpu_add(memcg->vmstats_percpu->events[i], count);
1088 memcg_rstat_updated(memcg, count, cpu);
1089 trace_count_memcg_events(memcg, idx, count);
1090
1091 put_cpu();
1092}
1093
1094unsigned long memcg_events(struct mem_cgroup *memcg, int event)
1095{
1096 int i = memcg_events_index(event);
1097
1098 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, event))
1099 return 0;
1100
1101 return READ_ONCE(memcg->vmstats->events[i]);
1102}
1103
1104bool memcg_vm_event_item_valid(enum vm_event_item idx)
1105{
1106 if (idx >= NR_VM_EVENT_ITEMS)
1107 return false;
1108
1109 return !BAD_STAT_IDX(memcg_events_index(idx));
1110}
1111
1112#ifdef CONFIG_MEMCG_V1
1113unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
1114{
1115 int i = memcg_events_index(event);
1116
1117 if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, event))
1118 return 0;
1119
1120 return READ_ONCE(memcg->vmstats->events_local[i]);
1121}
1122#endif
1123
1124struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1125{
1126 /*
1127 * mm_update_next_owner() may clear mm->owner to NULL
1128 * if it races with swapoff, page migration, etc.
1129 * So this can be called with p == NULL.
1130 */
1131 if (unlikely(!p))
1132 return NULL;
1133
1134 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
1135}
1136EXPORT_SYMBOL(mem_cgroup_from_task);
1137
1138static __always_inline struct mem_cgroup *active_memcg(void)
1139{
1140 if (!in_task())
1141 return this_cpu_read(int_active_memcg);
1142 else
1143 return current->active_memcg;
1144}
1145
1146/**
1147 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1148 * @mm: mm from which memcg should be extracted. It can be NULL.
1149 *
1150 * Obtain a reference on mm->memcg and returns it if successful. If mm
1151 * is NULL, then the memcg is chosen as follows:
1152 * 1) The active memcg, if set.
1153 * 2) current->mm->memcg, if available
1154 * 3) root memcg
1155 * If mem_cgroup is disabled, NULL is returned.
1156 */
1157struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
1158{
1159 struct mem_cgroup *memcg;
1160
1161 if (mem_cgroup_disabled())
1162 return NULL;
1163
1164 /*
1165 * Page cache insertions can happen without an
1166 * actual mm context, e.g. during disk probing
1167 * on boot, loopback IO, acct() writes etc.
1168 *
1169 * No need to css_get on root memcg as the reference
1170 * counting is disabled on the root level in the
1171 * cgroup core. See CSS_NO_REF.
1172 */
1173 if (unlikely(!mm)) {
1174 memcg = active_memcg();
1175 if (unlikely(memcg)) {
1176 /* remote memcg must hold a ref */
1177 css_get(&memcg->css);
1178 return memcg;
1179 }
1180 mm = current->mm;
1181 if (unlikely(!mm))
1182 return root_mem_cgroup;
1183 }
1184
1185 rcu_read_lock();
1186 do {
1187 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1188 if (unlikely(!memcg))
1189 memcg = root_mem_cgroup;
1190 } while (!css_tryget(&memcg->css));
1191 rcu_read_unlock();
1192 return memcg;
1193}
1194EXPORT_SYMBOL(get_mem_cgroup_from_mm);
1195
1196/**
1197 * get_mem_cgroup_from_current - Obtain a reference on current task's memcg.
1198 */
1199struct mem_cgroup *get_mem_cgroup_from_current(void)
1200{
1201 struct mem_cgroup *memcg;
1202
1203 if (mem_cgroup_disabled())
1204 return NULL;
1205
1206again:
1207 rcu_read_lock();
1208 memcg = mem_cgroup_from_task(current);
1209 if (!css_tryget(&memcg->css)) {
1210 rcu_read_unlock();
1211 goto again;
1212 }
1213 rcu_read_unlock();
1214 return memcg;
1215}
1216
1217/**
1218 * get_mem_cgroup_from_folio - Obtain a reference on a given folio's memcg.
1219 * @folio: folio from which memcg should be extracted.
1220 *
1221 * See folio_memcg() for folio->objcg/memcg binding rules.
1222 */
1223struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio)
1224{
1225 struct mem_cgroup *memcg;
1226
1227 if (mem_cgroup_disabled())
1228 return NULL;
1229
1230 if (!folio_memcg_charged(folio))
1231 return root_mem_cgroup;
1232
1233 rcu_read_lock();
1234 do {
1235 memcg = folio_memcg(folio);
1236 } while (unlikely(!css_tryget(&memcg->css)));
1237 rcu_read_unlock();
1238 return memcg;
1239}
1240
1241/**
1242 * mem_cgroup_iter - iterate over memory cgroup hierarchy
1243 * @root: hierarchy root
1244 * @prev: previously returned memcg, NULL on first invocation
1245 * @reclaim: cookie for shared reclaim walks, NULL for full walks
1246 *
1247 * Returns references to children of the hierarchy below @root, or
1248 * @root itself, or %NULL after a full round-trip.
1249 *
1250 * Caller must pass the return value in @prev on subsequent
1251 * invocations for reference counting, or use mem_cgroup_iter_break()
1252 * to cancel a hierarchy walk before the round-trip is complete.
1253 *
1254 * Reclaimers can specify a node in @reclaim to divide up the memcgs
1255 * in the hierarchy among all concurrent reclaimers operating on the
1256 * same node.
1257 */
1258struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1259 struct mem_cgroup *prev,
1260 struct mem_cgroup_reclaim_cookie *reclaim)
1261{
1262 struct mem_cgroup_reclaim_iter *iter;
1263 struct cgroup_subsys_state *css;
1264 struct mem_cgroup *pos;
1265 struct mem_cgroup *next;
1266
1267 if (mem_cgroup_disabled())
1268 return NULL;
1269
1270 if (!root)
1271 root = root_mem_cgroup;
1272
1273 rcu_read_lock();
1274restart:
1275 next = NULL;
1276
1277 if (reclaim) {
1278 int gen;
1279 int nid = reclaim->pgdat->node_id;
1280
1281 iter = &root->nodeinfo[nid]->iter;
1282 gen = atomic_read(&iter->generation);
1283
1284 /*
1285 * On start, join the current reclaim iteration cycle.
1286 * Exit when a concurrent walker completes it.
1287 */
1288 if (!prev)
1289 reclaim->generation = gen;
1290 else if (reclaim->generation != gen)
1291 goto out_unlock;
1292
1293 pos = READ_ONCE(iter->position);
1294 } else
1295 pos = prev;
1296
1297 css = pos ? &pos->css : NULL;
1298
1299 while ((css = css_next_descendant_pre(css, &root->css))) {
1300 /*
1301 * Verify the css and acquire a reference. The root
1302 * is provided by the caller, so we know it's alive
1303 * and kicking, and don't take an extra reference.
1304 */
1305 if (css == &root->css || css_tryget(css))
1306 break;
1307 }
1308
1309 next = mem_cgroup_from_css(css);
1310
1311 if (reclaim) {
1312 /*
1313 * The position could have already been updated by a competing
1314 * thread, so check that the value hasn't changed since we read
1315 * it to avoid reclaiming from the same cgroup twice.
1316 */
1317 if (cmpxchg(&iter->position, pos, next) != pos) {
1318 if (css && css != &root->css)
1319 css_put(css);
1320 goto restart;
1321 }
1322
1323 if (!next) {
1324 atomic_inc(&iter->generation);
1325
1326 /*
1327 * Reclaimers share the hierarchy walk, and a
1328 * new one might jump in right at the end of
1329 * the hierarchy - make sure they see at least
1330 * one group and restart from the beginning.
1331 */
1332 if (!prev)
1333 goto restart;
1334 }
1335 }
1336
1337out_unlock:
1338 rcu_read_unlock();
1339 if (prev && prev != root)
1340 css_put(&prev->css);
1341
1342 return next;
1343}
1344
1345/**
1346 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1347 * @root: hierarchy root
1348 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1349 */
1350void mem_cgroup_iter_break(struct mem_cgroup *root,
1351 struct mem_cgroup *prev)
1352{
1353 if (!root)
1354 root = root_mem_cgroup;
1355 if (prev && prev != root)
1356 css_put(&prev->css);
1357}
1358
1359static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1360 struct mem_cgroup *dead_memcg)
1361{
1362 struct mem_cgroup_reclaim_iter *iter;
1363 struct mem_cgroup_per_node *mz;
1364 int nid;
1365
1366 for_each_node(nid) {
1367 mz = from->nodeinfo[nid];
1368 iter = &mz->iter;
1369 cmpxchg(&iter->position, dead_memcg, NULL);
1370 }
1371}
1372
1373static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1374{
1375 struct mem_cgroup *memcg = dead_memcg;
1376 struct mem_cgroup *last;
1377
1378 do {
1379 __invalidate_reclaim_iterators(memcg, dead_memcg);
1380 last = memcg;
1381 } while ((memcg = parent_mem_cgroup(memcg)));
1382
1383 /*
1384 * When cgroup1 non-hierarchy mode is used,
1385 * parent_mem_cgroup() does not walk all the way up to the
1386 * cgroup root (root_mem_cgroup). So we have to handle
1387 * dead_memcg from cgroup root separately.
1388 */
1389 if (!mem_cgroup_is_root(last))
1390 __invalidate_reclaim_iterators(root_mem_cgroup,
1391 dead_memcg);
1392}
1393
1394/**
1395 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1396 * @memcg: hierarchy root
1397 * @fn: function to call for each task
1398 * @arg: argument passed to @fn
1399 *
1400 * This function iterates over tasks attached to @memcg or to any of its
1401 * descendants and calls @fn for each task. If @fn returns a non-zero
1402 * value, the function breaks the iteration loop. Otherwise, it will iterate
1403 * over all tasks and return 0.
1404 *
1405 * This function must not be called for the root memory cgroup.
1406 */
1407void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1408 int (*fn)(struct task_struct *, void *), void *arg)
1409{
1410 struct mem_cgroup *iter;
1411 int ret = 0;
1412
1413 BUG_ON(mem_cgroup_is_root(memcg));
1414
1415 for_each_mem_cgroup_tree(iter, memcg) {
1416 struct css_task_iter it;
1417 struct task_struct *task;
1418
1419 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
1420 while (!ret && (task = css_task_iter_next(&it))) {
1421 ret = fn(task, arg);
1422 /* Avoid potential softlockup warning */
1423 cond_resched();
1424 }
1425 css_task_iter_end(&it);
1426 if (ret) {
1427 mem_cgroup_iter_break(memcg, iter);
1428 break;
1429 }
1430 }
1431}
1432
1433/**
1434 * folio_lruvec_lock - Lock the lruvec for a folio.
1435 * @folio: Pointer to the folio.
1436 *
1437 * These functions are safe to use under any of the following conditions:
1438 * - folio locked
1439 * - folio_test_lru false
1440 * - folio frozen (refcount of 0)
1441 *
1442 * Return: The lruvec this folio is on with its lock held and rcu read lock held.
1443 */
1444struct lruvec *folio_lruvec_lock(struct folio *folio)
1445{
1446 struct lruvec *lruvec;
1447
1448 rcu_read_lock();
1449retry:
1450 lruvec = folio_lruvec(folio);
1451 spin_lock(&lruvec->lru_lock);
1452 if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
1453 spin_unlock(&lruvec->lru_lock);
1454 goto retry;
1455 }
1456
1457 return lruvec;
1458}
1459
1460/**
1461 * folio_lruvec_lock_irq - Lock the lruvec for a folio.
1462 * @folio: Pointer to the folio.
1463 *
1464 * These functions are safe to use under any of the following conditions:
1465 * - folio locked
1466 * - folio_test_lru false
1467 * - folio frozen (refcount of 0)
1468 *
1469 * Return: The lruvec this folio is on with its lock held and interrupts
1470 * disabled and rcu read lock held.
1471 */
1472struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
1473{
1474 struct lruvec *lruvec;
1475
1476 rcu_read_lock();
1477retry:
1478 lruvec = folio_lruvec(folio);
1479 spin_lock_irq(&lruvec->lru_lock);
1480 if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
1481 spin_unlock_irq(&lruvec->lru_lock);
1482 goto retry;
1483 }
1484
1485 return lruvec;
1486}
1487
1488/**
1489 * folio_lruvec_lock_irqsave - Lock the lruvec for a folio.
1490 * @folio: Pointer to the folio.
1491 * @flags: Pointer to irqsave flags.
1492 *
1493 * These functions are safe to use under any of the following conditions:
1494 * - folio locked
1495 * - folio_test_lru false
1496 * - folio frozen (refcount of 0)
1497 *
1498 * Return: The lruvec this folio is on with its lock held and interrupts
1499 * disabled and rcu read lock held.
1500 */
1501struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
1502 unsigned long *flags)
1503{
1504 struct lruvec *lruvec;
1505
1506 rcu_read_lock();
1507retry:
1508 lruvec = folio_lruvec(folio);
1509 spin_lock_irqsave(&lruvec->lru_lock, *flags);
1510 if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
1511 spin_unlock_irqrestore(&lruvec->lru_lock, *flags);
1512 goto retry;
1513 }
1514
1515 return lruvec;
1516}
1517
1518/**
1519 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1520 * @lruvec: mem_cgroup per zone lru vector
1521 * @lru: index of lru list the page is sitting on
1522 * @zid: zone id of the accounted pages
1523 * @nr_pages: positive when adding or negative when removing
1524 *
1525 * This function must be called under lru_lock, just before a page is added
1526 * to or just after a page is removed from an lru list.
1527 */
1528void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1529 int zid, long nr_pages)
1530{
1531 struct mem_cgroup_per_node *mz;
1532 unsigned long *lru_size;
1533 long size;
1534
1535 if (mem_cgroup_disabled())
1536 return;
1537
1538 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1539 lru_size = &mz->lru_zone_size[zid][lru];
1540
1541 if (nr_pages < 0)
1542 *lru_size += nr_pages;
1543
1544 size = *lru_size;
1545 if (WARN_ONCE(size < 0,
1546 "%s(%p, %d, %ld): lru_size %ld\n",
1547 __func__, lruvec, lru, nr_pages, size)) {
1548 VM_BUG_ON(1);
1549 *lru_size = 0;
1550 }
1551
1552 if (nr_pages > 0)
1553 *lru_size += nr_pages;
1554}
1555
1556/**
1557 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1558 * @memcg: the memory cgroup
1559 *
1560 * Returns the maximum amount of memory @mem can be charged with, in
1561 * pages.
1562 */
1563static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1564{
1565 unsigned long margin = 0;
1566 unsigned long count;
1567 unsigned long limit;
1568
1569 count = page_counter_read(&memcg->memory);
1570 limit = READ_ONCE(memcg->memory.max);
1571 if (count < limit)
1572 margin = limit - count;
1573
1574 if (do_memsw_account()) {
1575 count = page_counter_read(&memcg->memsw);
1576 limit = READ_ONCE(memcg->memsw.max);
1577 if (count < limit)
1578 margin = min(margin, limit - count);
1579 else
1580 margin = 0;
1581 }
1582
1583 return margin;
1584}
1585
1586struct memory_stat {
1587 const char *name;
1588 unsigned int idx;
1589};
1590
1591static const struct memory_stat memory_stats[] = {
1592 { "anon", NR_ANON_MAPPED },
1593 { "file", NR_FILE_PAGES },
1594 { "kernel", MEMCG_KMEM },
1595 { "kernel_stack", NR_KERNEL_STACK_KB },
1596 { "pagetables", NR_PAGETABLE },
1597 { "sec_pagetables", NR_SECONDARY_PAGETABLE },
1598 { "percpu", MEMCG_PERCPU_B },
1599 { "sock", MEMCG_SOCK },
1600 { "vmalloc", NR_VMALLOC },
1601 { "shmem", NR_SHMEM },
1602#ifdef CONFIG_ZSWAP
1603 { "zswap", MEMCG_ZSWAP_B },
1604 { "zswapped", MEMCG_ZSWAPPED },
1605 { "zswap_incomp", MEMCG_ZSWAP_INCOMP },
1606#endif
1607 { "file_mapped", NR_FILE_MAPPED },
1608 { "file_dirty", NR_FILE_DIRTY },
1609 { "file_writeback", NR_WRITEBACK },
1610#ifdef CONFIG_SWAP
1611 { "swapcached", NR_SWAPCACHE },
1612#endif
1613#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1614 { "anon_thp", NR_ANON_THPS },
1615 { "file_thp", NR_FILE_THPS },
1616 { "shmem_thp", NR_SHMEM_THPS },
1617#endif
1618 { "inactive_anon", NR_INACTIVE_ANON },
1619 { "active_anon", NR_ACTIVE_ANON },
1620 { "inactive_file", NR_INACTIVE_FILE },
1621 { "active_file", NR_ACTIVE_FILE },
1622 { "unevictable", NR_UNEVICTABLE },
1623 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B },
1624 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B },
1625#ifdef CONFIG_HUGETLB_PAGE
1626 { "hugetlb", NR_HUGETLB },
1627#endif
1628
1629 /* The memory events */
1630 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON },
1631 { "workingset_refault_file", WORKINGSET_REFAULT_FILE },
1632 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON },
1633 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE },
1634 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON },
1635 { "workingset_restore_file", WORKINGSET_RESTORE_FILE },
1636 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM },
1637
1638 { "pgdemote_kswapd", PGDEMOTE_KSWAPD },
1639 { "pgdemote_direct", PGDEMOTE_DIRECT },
1640 { "pgdemote_khugepaged", PGDEMOTE_KHUGEPAGED },
1641 { "pgdemote_proactive", PGDEMOTE_PROACTIVE },
1642 { "pgsteal_kswapd", PGSTEAL_KSWAPD },
1643 { "pgsteal_direct", PGSTEAL_DIRECT },
1644 { "pgsteal_khugepaged", PGSTEAL_KHUGEPAGED },
1645 { "pgsteal_proactive", PGSTEAL_PROACTIVE },
1646 { "pgscan_kswapd", PGSCAN_KSWAPD },
1647 { "pgscan_direct", PGSCAN_DIRECT },
1648 { "pgscan_khugepaged", PGSCAN_KHUGEPAGED },
1649 { "pgscan_proactive", PGSCAN_PROACTIVE },
1650 { "pgrefill", PGREFILL },
1651#ifdef CONFIG_NUMA_BALANCING
1652 { "pgpromote_success", PGPROMOTE_SUCCESS },
1653#endif
1654};
1655
1656/* The actual unit of the state item, not the same as the output unit */
1657static int memcg_page_state_unit(int item)
1658{
1659 switch (item) {
1660 case MEMCG_PERCPU_B:
1661 case MEMCG_ZSWAP_B:
1662 case NR_SLAB_RECLAIMABLE_B:
1663 case NR_SLAB_UNRECLAIMABLE_B:
1664 return 1;
1665 case NR_KERNEL_STACK_KB:
1666 return SZ_1K;
1667 default:
1668 return PAGE_SIZE;
1669 }
1670}
1671
1672/* Translate stat items to the correct unit for memory.stat output */
1673static int memcg_page_state_output_unit(int item)
1674{
1675 /*
1676 * Workingset state is actually in pages, but we export it to userspace
1677 * as a scalar count of events, so special case it here.
1678 *
1679 * Demotion and promotion activities are exported in pages, consistent
1680 * with their global counterparts.
1681 */
1682 switch (item) {
1683 case WORKINGSET_REFAULT_ANON:
1684 case WORKINGSET_REFAULT_FILE:
1685 case WORKINGSET_ACTIVATE_ANON:
1686 case WORKINGSET_ACTIVATE_FILE:
1687 case WORKINGSET_RESTORE_ANON:
1688 case WORKINGSET_RESTORE_FILE:
1689 case WORKINGSET_NODERECLAIM:
1690 case PGDEMOTE_KSWAPD:
1691 case PGDEMOTE_DIRECT:
1692 case PGDEMOTE_KHUGEPAGED:
1693 case PGDEMOTE_PROACTIVE:
1694 case PGSTEAL_KSWAPD:
1695 case PGSTEAL_DIRECT:
1696 case PGSTEAL_KHUGEPAGED:
1697 case PGSTEAL_PROACTIVE:
1698 case PGSCAN_KSWAPD:
1699 case PGSCAN_DIRECT:
1700 case PGSCAN_KHUGEPAGED:
1701 case PGSCAN_PROACTIVE:
1702 case PGREFILL:
1703#ifdef CONFIG_NUMA_BALANCING
1704 case PGPROMOTE_SUCCESS:
1705#endif
1706 return 1;
1707 default:
1708 return memcg_page_state_unit(item);
1709 }
1710}
1711
1712unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item)
1713{
1714 return memcg_page_state(memcg, item) *
1715 memcg_page_state_output_unit(item);
1716}
1717
1718#ifdef CONFIG_MEMCG_V1
1719unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item)
1720{
1721 return memcg_page_state_local(memcg, item) *
1722 memcg_page_state_output_unit(item);
1723}
1724#endif
1725
1726#ifdef CONFIG_HUGETLB_PAGE
1727static bool memcg_accounts_hugetlb(void)
1728{
1729 return cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_HUGETLB_ACCOUNTING;
1730}
1731#else /* CONFIG_HUGETLB_PAGE */
1732static bool memcg_accounts_hugetlb(void)
1733{
1734 return false;
1735}
1736#endif /* CONFIG_HUGETLB_PAGE */
1737
1738static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1739{
1740 int i;
1741
1742 /*
1743 * Provide statistics on the state of the memory subsystem as
1744 * well as cumulative event counters that show past behavior.
1745 *
1746 * This list is ordered following a combination of these gradients:
1747 * 1) generic big picture -> specifics and details
1748 * 2) reflecting userspace activity -> reflecting kernel heuristics
1749 *
1750 * Current memory state:
1751 */
1752 mem_cgroup_flush_stats(memcg);
1753
1754 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
1755 u64 size;
1756
1757#ifdef CONFIG_HUGETLB_PAGE
1758 if (unlikely(memory_stats[i].idx == NR_HUGETLB) &&
1759 !memcg_accounts_hugetlb())
1760 continue;
1761#endif
1762 size = memcg_page_state_output(memcg, memory_stats[i].idx);
1763 seq_buf_printf(s, "%s %llu\n", memory_stats[i].name, size);
1764
1765 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
1766 size += memcg_page_state_output(memcg,
1767 NR_SLAB_RECLAIMABLE_B);
1768 seq_buf_printf(s, "slab %llu\n", size);
1769 }
1770 }
1771
1772 /* Accumulated memory events */
1773 seq_buf_printf(s, "pgscan %lu\n",
1774 memcg_page_state(memcg, PGSCAN_KSWAPD) +
1775 memcg_page_state(memcg, PGSCAN_DIRECT) +
1776 memcg_page_state(memcg, PGSCAN_PROACTIVE) +
1777 memcg_page_state(memcg, PGSCAN_KHUGEPAGED));
1778 seq_buf_printf(s, "pgsteal %lu\n",
1779 memcg_page_state(memcg, PGSTEAL_KSWAPD) +
1780 memcg_page_state(memcg, PGSTEAL_DIRECT) +
1781 memcg_page_state(memcg, PGSTEAL_PROACTIVE) +
1782 memcg_page_state(memcg, PGSTEAL_KHUGEPAGED));
1783
1784 for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {
1785#ifdef CONFIG_MEMCG_V1
1786 if (memcg_vm_event_stat[i] == PGPGIN ||
1787 memcg_vm_event_stat[i] == PGPGOUT)
1788 continue;
1789#endif
1790 seq_buf_printf(s, "%s %lu\n",
1791 vm_event_name(memcg_vm_event_stat[i]),
1792 memcg_events(memcg, memcg_vm_event_stat[i]));
1793 }
1794}
1795
1796static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
1797{
1798 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1799 memcg_stat_format(memcg, s);
1800 else
1801 memcg1_stat_format(memcg, s);
1802 if (seq_buf_has_overflowed(s))
1803 pr_warn("%s: Warning, stat buffer overflow, please report\n", __func__);
1804}
1805
1806/**
1807 * mem_cgroup_print_oom_context: Print OOM information relevant to
1808 * memory controller.
1809 * @memcg: The memory cgroup that went over limit
1810 * @p: Task that is going to be killed
1811 *
1812 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1813 * enabled
1814 */
1815void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1816{
1817 rcu_read_lock();
1818
1819 if (memcg) {
1820 pr_cont(",oom_memcg=");
1821 pr_cont_cgroup_path(memcg->css.cgroup);
1822 } else
1823 pr_cont(",global_oom");
1824 if (p) {
1825 pr_cont(",task_memcg=");
1826 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1827 }
1828 rcu_read_unlock();
1829}
1830
1831/**
1832 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1833 * memory controller.
1834 * @memcg: The memory cgroup that went over limit
1835 */
1836void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1837{
1838 /* Use static buffer, for the caller is holding oom_lock. */
1839 static char buf[SEQ_BUF_SIZE];
1840 struct seq_buf s;
1841 unsigned long memory_failcnt;
1842
1843 lockdep_assert_held(&oom_lock);
1844
1845 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1846 memory_failcnt = atomic_long_read(&memcg->memory_events[MEMCG_MAX]);
1847 else
1848 memory_failcnt = memcg->memory.failcnt;
1849
1850 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1851 K((u64)page_counter_read(&memcg->memory)),
1852 K((u64)READ_ONCE(memcg->memory.max)), memory_failcnt);
1853 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1854 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1855 K((u64)page_counter_read(&memcg->swap)),
1856 K((u64)READ_ONCE(memcg->swap.max)),
1857 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
1858#ifdef CONFIG_MEMCG_V1
1859 else {
1860 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1861 K((u64)page_counter_read(&memcg->memsw)),
1862 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1863 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1864 K((u64)page_counter_read(&memcg->kmem)),
1865 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1866 }
1867#endif
1868
1869 pr_info("Memory cgroup stats for ");
1870 pr_cont_cgroup_path(memcg->css.cgroup);
1871 pr_cont(":");
1872 seq_buf_init(&s, buf, SEQ_BUF_SIZE);
1873 memory_stat_format(memcg, &s);
1874 seq_buf_do_printk(&s, KERN_INFO);
1875}
1876
1877/*
1878 * Return the memory (and swap, if configured) limit for a memcg.
1879 */
1880unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1881{
1882 unsigned long max = READ_ONCE(memcg->memory.max);
1883
1884 if (do_memsw_account()) {
1885 if (mem_cgroup_swappiness(memcg)) {
1886 /* Calculate swap excess capacity from memsw limit */
1887 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
1888
1889 max += min(swap, (unsigned long)total_swap_pages);
1890 }
1891 } else {
1892 if (mem_cgroup_swappiness(memcg))
1893 max += min(READ_ONCE(memcg->swap.max),
1894 (unsigned long)total_swap_pages);
1895 }
1896 return max;
1897}
1898
1899void __memcg_memory_event(struct mem_cgroup *memcg,
1900 enum memcg_memory_event event, bool allow_spinning)
1901{
1902 bool swap_event = event == MEMCG_SWAP_HIGH || event == MEMCG_SWAP_MAX ||
1903 event == MEMCG_SWAP_FAIL;
1904
1905 /* For now only MEMCG_MAX can happen with !allow_spinning context. */
1906 VM_WARN_ON_ONCE(!allow_spinning && event != MEMCG_MAX);
1907
1908 atomic_long_inc(&memcg->memory_events_local[event]);
1909 if (!swap_event && allow_spinning)
1910 cgroup_file_notify(&memcg->events_local_file);
1911
1912 do {
1913 atomic_long_inc(&memcg->memory_events[event]);
1914 if (allow_spinning) {
1915 if (swap_event)
1916 cgroup_file_notify(&memcg->swap_events_file);
1917 else
1918 cgroup_file_notify(&memcg->events_file);
1919 }
1920
1921 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1922 break;
1923 if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS)
1924 break;
1925 } while ((memcg = parent_mem_cgroup(memcg)) &&
1926 !mem_cgroup_is_root(memcg));
1927}
1928EXPORT_SYMBOL_GPL(__memcg_memory_event);
1929
1930static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1931 int order)
1932{
1933 struct oom_control oc = {
1934 .zonelist = NULL,
1935 .nodemask = NULL,
1936 .memcg = memcg,
1937 .gfp_mask = gfp_mask,
1938 .order = order,
1939 };
1940 bool ret = true;
1941
1942 if (mutex_lock_killable(&oom_lock))
1943 return true;
1944
1945 if (mem_cgroup_margin(memcg) >= (1 << order))
1946 goto unlock;
1947
1948 /*
1949 * A few threads which were not waiting at mutex_lock_killable() can
1950 * fail to bail out. Therefore, check again after holding oom_lock.
1951 */
1952 ret = out_of_memory(&oc);
1953
1954unlock:
1955 mutex_unlock(&oom_lock);
1956 return ret;
1957}
1958
1959/*
1960 * Returns true if successfully killed one or more processes. Though in some
1961 * corner cases it can return true even without killing any process.
1962 */
1963static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1964{
1965 bool locked, ret;
1966
1967 if (order > PAGE_ALLOC_COSTLY_ORDER)
1968 return false;
1969
1970 memcg_memory_event(memcg, MEMCG_OOM);
1971
1972 if (!memcg1_oom_prepare(memcg, &locked))
1973 return false;
1974
1975 ret = mem_cgroup_out_of_memory(memcg, mask, order);
1976
1977 memcg1_oom_finish(memcg, locked);
1978
1979 return ret;
1980}
1981
1982/**
1983 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
1984 * @victim: task to be killed by the OOM killer
1985 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
1986 *
1987 * Returns a pointer to a memory cgroup, which has to be cleaned up
1988 * by killing all belonging OOM-killable tasks.
1989 *
1990 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
1991 */
1992struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
1993 struct mem_cgroup *oom_domain)
1994{
1995 struct mem_cgroup *oom_group = NULL;
1996 struct mem_cgroup *memcg;
1997
1998 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1999 return NULL;
2000
2001 if (!oom_domain)
2002 oom_domain = root_mem_cgroup;
2003
2004 rcu_read_lock();
2005
2006 memcg = mem_cgroup_from_task(victim);
2007 if (mem_cgroup_is_root(memcg))
2008 goto out;
2009
2010 /*
2011 * If the victim task has been asynchronously moved to a different
2012 * memory cgroup, we might end up killing tasks outside oom_domain.
2013 * In this case it's better to ignore memory.group.oom.
2014 */
2015 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2016 goto out;
2017
2018 /*
2019 * Traverse the memory cgroup hierarchy from the victim task's
2020 * cgroup up to the OOMing cgroup (or root) to find the
2021 * highest-level memory cgroup with oom.group set.
2022 */
2023 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2024 if (READ_ONCE(memcg->oom_group))
2025 oom_group = memcg;
2026
2027 if (memcg == oom_domain)
2028 break;
2029 }
2030
2031 if (oom_group)
2032 css_get(&oom_group->css);
2033out:
2034 rcu_read_unlock();
2035
2036 return oom_group;
2037}
2038
2039void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2040{
2041 pr_info("Tasks in ");
2042 pr_cont_cgroup_path(memcg->css.cgroup);
2043 pr_cont(" are going to be killed due to memory.oom.group set\n");
2044}
2045
2046/*
2047 * The value of NR_MEMCG_STOCK is selected to keep the cached memcgs and their
2048 * nr_pages in a single cacheline. This may change in future.
2049 */
2050#define NR_MEMCG_STOCK 7
2051#define FLUSHING_CACHED_CHARGE 0
2052struct memcg_stock_pcp {
2053 local_trylock_t lock;
2054 uint8_t nr_pages[NR_MEMCG_STOCK];
2055 struct mem_cgroup *cached[NR_MEMCG_STOCK];
2056
2057 struct work_struct work;
2058 unsigned long flags;
2059 uint8_t drain_idx;
2060};
2061
2062static DEFINE_PER_CPU_ALIGNED(struct memcg_stock_pcp, memcg_stock) = {
2063 .lock = INIT_LOCAL_TRYLOCK(lock),
2064};
2065
2066/*
2067 * NR_OBJ_STOCK is sized so the entire hot path of obj_stock_pcp
2068 * (lock, accounting metadata, nr_bytes[] and cached[]) fits within a
2069 * single 64-byte cache line on non-debug 64-bit builds. With 5 slots:
2070 * lock(1) + index(1) + node_id(2) + slab stats(4) + nr_bytes(10)
2071 * + pad(6) + cached(40) == 64 bytes.
2072 * A CPU can thus consume/refill/account against five different objcgs
2073 * (typically per-node variants of the same memcg) while incurring at
2074 * most one cache miss on the stock.
2075 */
2076#define NR_OBJ_STOCK 5
2077struct obj_stock_pcp {
2078 local_trylock_t lock;
2079 int8_t index;
2080 int16_t node_id;
2081 int16_t nr_slab_reclaimable_b;
2082 int16_t nr_slab_unreclaimable_b;
2083#if PAGE_SHIFT > 16
2084 /*
2085 * On rare archs with 256KiB base page size (hexagon and powerpc 44x)
2086 * keep nr_bytes to unsigned int as uint16_t cannot represent the full
2087 * sub-page remainder. Such archs are not cacheline optimization targets.
2088 */
2089 unsigned int nr_bytes[NR_OBJ_STOCK];
2090#else
2091 uint16_t nr_bytes[NR_OBJ_STOCK];
2092#endif
2093 struct obj_cgroup *cached[NR_OBJ_STOCK];
2094
2095 struct work_struct work;
2096 unsigned long flags;
2097 uint8_t drain_idx;
2098};
2099
2100static DEFINE_PER_CPU_ALIGNED(struct obj_stock_pcp, obj_stock) = {
2101 .lock = INIT_LOCAL_TRYLOCK(lock),
2102 .index = -1,
2103 .node_id = NUMA_NO_NODE,
2104};
2105
2106static DEFINE_MUTEX(percpu_charge_mutex);
2107
2108static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i);
2109static void drain_obj_stock(struct obj_stock_pcp *stock);
2110static bool obj_stock_flush_required(struct obj_stock_pcp *stock,
2111 struct mem_cgroup *root_memcg);
2112
2113/**
2114 * consume_stock: Try to consume stocked charge on this cpu.
2115 * @memcg: memcg to consume from.
2116 * @nr_pages: how many pages to charge.
2117 *
2118 * Consume the cached charge if enough nr_pages are present otherwise return
2119 * failure. Also return failure for charge request larger than
2120 * MEMCG_CHARGE_BATCH or if the local lock is already taken.
2121 *
2122 * returns true if successful, false otherwise.
2123 */
2124static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2125{
2126 struct memcg_stock_pcp *stock;
2127 uint8_t stock_pages;
2128 bool ret = false;
2129 int i;
2130
2131 if (nr_pages > MEMCG_CHARGE_BATCH ||
2132 !local_trylock(&memcg_stock.lock))
2133 return ret;
2134
2135 stock = this_cpu_ptr(&memcg_stock);
2136
2137 for (i = 0; i < NR_MEMCG_STOCK; ++i) {
2138 if (memcg != READ_ONCE(stock->cached[i]))
2139 continue;
2140
2141 stock_pages = READ_ONCE(stock->nr_pages[i]);
2142 if (stock_pages >= nr_pages) {
2143 stock_pages -= nr_pages;
2144 WRITE_ONCE(stock->nr_pages[i], stock_pages);
2145 if (!stock_pages) {
2146 css_put(&memcg->css);
2147 WRITE_ONCE(stock->cached[i], NULL);
2148 }
2149 ret = true;
2150 }
2151 break;
2152 }
2153
2154 local_unlock(&memcg_stock.lock);
2155
2156 return ret;
2157}
2158
2159static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
2160{
2161 page_counter_uncharge(&memcg->memory, nr_pages);
2162 if (do_memsw_account())
2163 page_counter_uncharge(&memcg->memsw, nr_pages);
2164}
2165
2166/*
2167 * Returns stocks cached in percpu and reset cached information.
2168 */
2169static void drain_stock(struct memcg_stock_pcp *stock, int i)
2170{
2171 struct mem_cgroup *old = READ_ONCE(stock->cached[i]);
2172 uint8_t stock_pages;
2173
2174 if (!old)
2175 return;
2176
2177 stock_pages = READ_ONCE(stock->nr_pages[i]);
2178 if (stock_pages) {
2179 memcg_uncharge(old, stock_pages);
2180 WRITE_ONCE(stock->nr_pages[i], 0);
2181 }
2182
2183 css_put(&old->css);
2184 WRITE_ONCE(stock->cached[i], NULL);
2185}
2186
2187static void drain_stock_fully(struct memcg_stock_pcp *stock)
2188{
2189 int i;
2190
2191 for (i = 0; i < NR_MEMCG_STOCK; ++i)
2192 drain_stock(stock, i);
2193}
2194
2195static void drain_local_memcg_stock(struct work_struct *dummy)
2196{
2197 struct memcg_stock_pcp *stock;
2198
2199 if (WARN_ONCE(!in_task(), "drain in non-task context"))
2200 return;
2201
2202 local_lock(&memcg_stock.lock);
2203
2204 stock = this_cpu_ptr(&memcg_stock);
2205 drain_stock_fully(stock);
2206 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2207
2208 local_unlock(&memcg_stock.lock);
2209}
2210
2211static void drain_local_obj_stock(struct work_struct *dummy)
2212{
2213 struct obj_stock_pcp *stock;
2214
2215 if (WARN_ONCE(!in_task(), "drain in non-task context"))
2216 return;
2217
2218 local_lock(&obj_stock.lock);
2219
2220 stock = this_cpu_ptr(&obj_stock);
2221 drain_obj_stock(stock);
2222 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2223
2224 local_unlock(&obj_stock.lock);
2225}
2226
2227static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2228{
2229 struct memcg_stock_pcp *stock;
2230 struct mem_cgroup *cached;
2231 uint8_t stock_pages;
2232 bool success = false;
2233 int empty_slot = -1;
2234 int i;
2235
2236 /*
2237 * For now limit MEMCG_CHARGE_BATCH to 127 and less. In future if we
2238 * decide to increase it more than 127 then we will need more careful
2239 * handling of nr_pages[] in struct memcg_stock_pcp.
2240 */
2241 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX);
2242
2243 VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg));
2244
2245 if (nr_pages > MEMCG_CHARGE_BATCH ||
2246 !local_trylock(&memcg_stock.lock)) {
2247 /*
2248 * In case of larger than batch refill or unlikely failure to
2249 * lock the percpu memcg_stock.lock, uncharge memcg directly.
2250 */
2251 memcg_uncharge(memcg, nr_pages);
2252 return;
2253 }
2254
2255 stock = this_cpu_ptr(&memcg_stock);
2256 for (i = 0; i < NR_MEMCG_STOCK; ++i) {
2257 cached = READ_ONCE(stock->cached[i]);
2258 if (!cached && empty_slot == -1)
2259 empty_slot = i;
2260 if (memcg == READ_ONCE(stock->cached[i])) {
2261 stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages;
2262 WRITE_ONCE(stock->nr_pages[i], stock_pages);
2263 if (stock_pages > MEMCG_CHARGE_BATCH)
2264 drain_stock(stock, i);
2265 success = true;
2266 break;
2267 }
2268 }
2269
2270 if (!success) {
2271 i = empty_slot;
2272 if (i == -1) {
2273 i = stock->drain_idx++;
2274 if (stock->drain_idx == NR_MEMCG_STOCK)
2275 stock->drain_idx = 0;
2276 drain_stock(stock, i);
2277 }
2278 css_get(&memcg->css);
2279 WRITE_ONCE(stock->cached[i], memcg);
2280 WRITE_ONCE(stock->nr_pages[i], nr_pages);
2281 }
2282
2283 local_unlock(&memcg_stock.lock);
2284}
2285
2286static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock,
2287 struct mem_cgroup *root_memcg)
2288{
2289 struct mem_cgroup *memcg;
2290 bool flush = false;
2291 int i;
2292
2293 rcu_read_lock();
2294 for (i = 0; i < NR_MEMCG_STOCK; ++i) {
2295 memcg = READ_ONCE(stock->cached[i]);
2296 if (!memcg)
2297 continue;
2298
2299 if (READ_ONCE(stock->nr_pages[i]) &&
2300 mem_cgroup_is_descendant(memcg, root_memcg)) {
2301 flush = true;
2302 break;
2303 }
2304 }
2305 rcu_read_unlock();
2306 return flush;
2307}
2308
2309static void schedule_drain_work(int cpu, struct work_struct *work)
2310{
2311 /*
2312 * Protect housekeeping cpumask read and work enqueue together
2313 * in the same RCU critical section so that later cpuset isolated
2314 * partition update only need to wait for an RCU GP and flush the
2315 * pending work on newly isolated CPUs.
2316 */
2317 guard(rcu)();
2318 if (!cpu_is_isolated(cpu))
2319 queue_work_on(cpu, memcg_wq, work);
2320}
2321
2322/*
2323 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2324 * of the hierarchy under it.
2325 */
2326void drain_all_stock(struct mem_cgroup *root_memcg)
2327{
2328 int cpu, curcpu;
2329
2330 /* If someone's already draining, avoid adding running more workers. */
2331 if (!mutex_trylock(&percpu_charge_mutex))
2332 return;
2333 /*
2334 * Notify other cpus that system-wide "drain" is running
2335 * We do not care about races with the cpu hotplug because cpu down
2336 * as well as workers from this path always operate on the local
2337 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2338 */
2339 migrate_disable();
2340 curcpu = smp_processor_id();
2341 for_each_online_cpu(cpu) {
2342 struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu);
2343 struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu);
2344
2345 if (!test_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags) &&
2346 is_memcg_drain_needed(memcg_st, root_memcg) &&
2347 !test_and_set_bit(FLUSHING_CACHED_CHARGE,
2348 &memcg_st->flags)) {
2349 if (cpu == curcpu)
2350 drain_local_memcg_stock(&memcg_st->work);
2351 else
2352 schedule_drain_work(cpu, &memcg_st->work);
2353 }
2354
2355 if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) &&
2356 obj_stock_flush_required(obj_st, root_memcg) &&
2357 !test_and_set_bit(FLUSHING_CACHED_CHARGE,
2358 &obj_st->flags)) {
2359 if (cpu == curcpu)
2360 drain_local_obj_stock(&obj_st->work);
2361 else
2362 schedule_drain_work(cpu, &obj_st->work);
2363 }
2364 }
2365 migrate_enable();
2366 mutex_unlock(&percpu_charge_mutex);
2367}
2368
2369static int memcg_hotplug_cpu_dead(unsigned int cpu)
2370{
2371 /* no need for the local lock */
2372 drain_obj_stock(&per_cpu(obj_stock, cpu));
2373 drain_stock_fully(&per_cpu(memcg_stock, cpu));
2374
2375 return 0;
2376}
2377
2378static unsigned long reclaim_high(struct mem_cgroup *memcg,
2379 unsigned int nr_pages,
2380 gfp_t gfp_mask)
2381{
2382 unsigned long nr_reclaimed = 0;
2383
2384 do {
2385 unsigned long pflags;
2386
2387 if (page_counter_read(&memcg->memory) <=
2388 READ_ONCE(memcg->memory.high))
2389 continue;
2390
2391 memcg_memory_event(memcg, MEMCG_HIGH);
2392
2393 psi_memstall_enter(&pflags);
2394 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2395 gfp_mask,
2396 MEMCG_RECLAIM_MAY_SWAP,
2397 NULL);
2398 psi_memstall_leave(&pflags);
2399 } while ((memcg = parent_mem_cgroup(memcg)) &&
2400 !mem_cgroup_is_root(memcg));
2401
2402 return nr_reclaimed;
2403}
2404
2405static void high_work_func(struct work_struct *work)
2406{
2407 struct mem_cgroup *memcg;
2408
2409 memcg = container_of(work, struct mem_cgroup, high_work);
2410 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2411}
2412
2413/*
2414 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
2415 * enough to still cause a significant slowdown in most cases, while still
2416 * allowing diagnostics and tracing to proceed without becoming stuck.
2417 */
2418#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2419
2420/*
2421 * When calculating the delay, we use these either side of the exponentiation to
2422 * maintain precision and scale to a reasonable number of jiffies (see the table
2423 * below.
2424 *
2425 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2426 * overage ratio to a delay.
2427 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
2428 * proposed penalty in order to reduce to a reasonable number of jiffies, and
2429 * to produce a reasonable delay curve.
2430 *
2431 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
2432 * reasonable delay curve compared to precision-adjusted overage, not
2433 * penalising heavily at first, but still making sure that growth beyond the
2434 * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2435 * example, with a high of 100 megabytes:
2436 *
2437 * +-------+------------------------+
2438 * | usage | time to allocate in ms |
2439 * +-------+------------------------+
2440 * | 100M | 0 |
2441 * | 101M | 6 |
2442 * | 102M | 25 |
2443 * | 103M | 57 |
2444 * | 104M | 102 |
2445 * | 105M | 159 |
2446 * | 106M | 230 |
2447 * | 107M | 313 |
2448 * | 108M | 409 |
2449 * | 109M | 518 |
2450 * | 110M | 639 |
2451 * | 111M | 774 |
2452 * | 112M | 921 |
2453 * | 113M | 1081 |
2454 * | 114M | 1254 |
2455 * | 115M | 1439 |
2456 * | 116M | 1638 |
2457 * | 117M | 1849 |
2458 * | 118M | 2000 |
2459 * | 119M | 2000 |
2460 * | 120M | 2000 |
2461 * +-------+------------------------+
2462 */
2463 #define MEMCG_DELAY_PRECISION_SHIFT 20
2464 #define MEMCG_DELAY_SCALING_SHIFT 14
2465
2466static u64 calculate_overage(unsigned long usage, unsigned long high)
2467{
2468 u64 overage;
2469
2470 if (usage <= high)
2471 return 0;
2472
2473 /*
2474 * Prevent division by 0 in overage calculation by acting as if
2475 * it was a threshold of 1 page
2476 */
2477 high = max(high, 1UL);
2478
2479 overage = usage - high;
2480 overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2481 return div64_u64(overage, high);
2482}
2483
2484static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2485{
2486 u64 overage, max_overage = 0;
2487
2488 do {
2489 overage = calculate_overage(page_counter_read(&memcg->memory),
2490 READ_ONCE(memcg->memory.high));
2491 max_overage = max(overage, max_overage);
2492 } while ((memcg = parent_mem_cgroup(memcg)) &&
2493 !mem_cgroup_is_root(memcg));
2494
2495 return max_overage;
2496}
2497
2498static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2499{
2500 u64 overage, max_overage = 0;
2501
2502 do {
2503 overage = calculate_overage(page_counter_read(&memcg->swap),
2504 READ_ONCE(memcg->swap.high));
2505 if (overage)
2506 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2507 max_overage = max(overage, max_overage);
2508 } while ((memcg = parent_mem_cgroup(memcg)) &&
2509 !mem_cgroup_is_root(memcg));
2510
2511 return max_overage;
2512}
2513
2514/*
2515 * Get the number of jiffies that we should penalise a mischievous cgroup which
2516 * is exceeding its memory.high by checking both it and its ancestors.
2517 */
2518static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
2519 unsigned int nr_pages,
2520 u64 max_overage)
2521{
2522 unsigned long penalty_jiffies;
2523
2524 if (!max_overage)
2525 return 0;
2526
2527 /*
2528 * We use overage compared to memory.high to calculate the number of
2529 * jiffies to sleep (penalty_jiffies). Ideally this value should be
2530 * fairly lenient on small overages, and increasingly harsh when the
2531 * memcg in question makes it clear that it has no intention of stopping
2532 * its crazy behaviour, so we exponentially increase the delay based on
2533 * overage amount.
2534 */
2535 penalty_jiffies = max_overage * max_overage * HZ;
2536 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2537 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
2538
2539 /*
2540 * Factor in the task's own contribution to the overage, such that four
2541 * N-sized allocations are throttled approximately the same as one
2542 * 4N-sized allocation.
2543 *
2544 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
2545 * larger the current charge patch is than that.
2546 */
2547 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
2548}
2549
2550/*
2551 * Reclaims memory over the high limit. Called directly from
2552 * try_charge() (context permitting), as well as from the userland
2553 * return path where reclaim is always able to block.
2554 */
2555void __mem_cgroup_handle_over_high(gfp_t gfp_mask)
2556{
2557 unsigned long penalty_jiffies;
2558 unsigned long pflags;
2559 unsigned long nr_reclaimed;
2560 unsigned int nr_pages = current->memcg_nr_pages_over_high;
2561 int nr_retries = MAX_RECLAIM_RETRIES;
2562 struct mem_cgroup *memcg;
2563 bool in_retry = false;
2564
2565 memcg = get_mem_cgroup_from_mm(current->mm);
2566 current->memcg_nr_pages_over_high = 0;
2567
2568retry_reclaim:
2569 /*
2570 * Bail if the task is already exiting. Unlike memory.max,
2571 * memory.high enforcement isn't as strict, and there is no
2572 * OOM killer involved, which means the excess could already
2573 * be much bigger (and still growing) than it could for
2574 * memory.max; the dying task could get stuck in fruitless
2575 * reclaim for a long time, which isn't desirable.
2576 */
2577 if (task_is_dying())
2578 goto out;
2579
2580 /*
2581 * The allocating task should reclaim at least the batch size, but for
2582 * subsequent retries we only want to do what's necessary to prevent oom
2583 * or breaching resource isolation.
2584 *
2585 * This is distinct from memory.max or page allocator behaviour because
2586 * memory.high is currently batched, whereas memory.max and the page
2587 * allocator run every time an allocation is made.
2588 */
2589 nr_reclaimed = reclaim_high(memcg,
2590 in_retry ? SWAP_CLUSTER_MAX : nr_pages,
2591 gfp_mask);
2592
2593 /*
2594 * memory.high is breached and reclaim is unable to keep up. Throttle
2595 * allocators proactively to slow down excessive growth.
2596 */
2597 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2598 mem_find_max_overage(memcg));
2599
2600 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2601 swap_find_max_overage(memcg));
2602
2603 /*
2604 * Clamp the max delay per usermode return so as to still keep the
2605 * application moving forwards and also permit diagnostics, albeit
2606 * extremely slowly.
2607 */
2608 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2609
2610 /*
2611 * Don't sleep if the amount of jiffies this memcg owes us is so low
2612 * that it's not even worth doing, in an attempt to be nice to those who
2613 * go only a small amount over their memory.high value and maybe haven't
2614 * been aggressively reclaimed enough yet.
2615 */
2616 if (penalty_jiffies <= HZ / 100)
2617 goto out;
2618
2619 /*
2620 * If reclaim is making forward progress but we're still over
2621 * memory.high, we want to encourage that rather than doing allocator
2622 * throttling.
2623 */
2624 if (nr_reclaimed || nr_retries--) {
2625 in_retry = true;
2626 goto retry_reclaim;
2627 }
2628
2629 /*
2630 * Reclaim didn't manage to push usage below the limit, slow
2631 * this allocating task down.
2632 *
2633 * If we exit early, we're guaranteed to die (since
2634 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
2635 * need to account for any ill-begotten jiffies to pay them off later.
2636 */
2637 psi_memstall_enter(&pflags);
2638 schedule_timeout_killable(penalty_jiffies);
2639 psi_memstall_leave(&pflags);
2640
2641out:
2642 css_put(&memcg->css);
2643}
2644
2645static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
2646 unsigned int nr_pages)
2647{
2648 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2649 int nr_retries = MAX_RECLAIM_RETRIES;
2650 struct mem_cgroup *mem_over_limit;
2651 struct page_counter *counter;
2652 unsigned long nr_reclaimed;
2653 bool passed_oom = false;
2654 unsigned int reclaim_options;
2655 bool drained = false;
2656 bool raised_max_event = false;
2657 unsigned long pflags;
2658 bool allow_spinning = gfpflags_allow_spinning(gfp_mask);
2659
2660retry:
2661 if (consume_stock(memcg, nr_pages))
2662 return 0;
2663
2664 if (!allow_spinning)
2665 /* Avoid the refill and flush of the older stock */
2666 batch = nr_pages;
2667
2668 reclaim_options = MEMCG_RECLAIM_MAY_SWAP;
2669 if (!do_memsw_account() ||
2670 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2671 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2672 goto done_restock;
2673 if (do_memsw_account())
2674 page_counter_uncharge(&memcg->memsw, batch);
2675 mem_over_limit = mem_cgroup_from_counter(counter, memory);
2676 } else {
2677 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2678 reclaim_options &= ~MEMCG_RECLAIM_MAY_SWAP;
2679 }
2680
2681 if (batch > nr_pages) {
2682 batch = nr_pages;
2683 goto retry;
2684 }
2685
2686 /*
2687 * Prevent unbounded recursion when reclaim operations need to
2688 * allocate memory. This might exceed the limits temporarily,
2689 * but we prefer facilitating memory reclaim and getting back
2690 * under the limit over triggering OOM kills in these cases.
2691 */
2692 if (unlikely(current->flags & PF_MEMALLOC))
2693 goto force;
2694
2695 if (unlikely(task_in_memcg_oom(current)))
2696 goto nomem;
2697
2698 if (!gfpflags_allow_blocking(gfp_mask))
2699 goto nomem;
2700
2701 /*
2702 * OOM victim still needs to charge memory to exit. OOM reaper should
2703 * help but it might fail on mmap_lock contention. If the victim is a
2704 * large thread group then all exiting threads might compete on oom_lock
2705 * just to learn that there is nothing really killable anymore. Bail
2706 * out early and fail the charge to expedite their exit. They are
2707 * considered fully reclaimed by the oom reaper and they shouldn't
2708 * contribute further charges.
2709 */
2710 if (tsk_is_oom_victim(current) &&
2711 mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm))
2712 goto nomem;
2713
2714 __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
2715 raised_max_event = true;
2716
2717 psi_memstall_enter(&pflags);
2718 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2719 gfp_mask, reclaim_options, NULL);
2720 psi_memstall_leave(&pflags);
2721
2722 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2723 goto retry;
2724
2725 if (!drained) {
2726 drain_all_stock(mem_over_limit);
2727 drained = true;
2728 goto retry;
2729 }
2730
2731 if (gfp_mask & __GFP_NORETRY)
2732 goto nomem;
2733 /*
2734 * Even though the limit is exceeded at this point, reclaim
2735 * may have been able to free some pages. Retry the charge
2736 * before killing the task.
2737 *
2738 * Only for regular pages, though: huge pages are rather
2739 * unlikely to succeed so close to the limit, and we fall back
2740 * to regular pages anyway in case of failure.
2741 */
2742 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2743 goto retry;
2744
2745 if (nr_retries--)
2746 goto retry;
2747
2748 if (gfp_mask & __GFP_RETRY_MAYFAIL)
2749 goto nomem;
2750
2751 /* Avoid endless loop for tasks bypassed by the oom killer */
2752 if (passed_oom && task_is_dying())
2753 goto nomem;
2754
2755 /*
2756 * keep retrying as long as the memcg oom killer is able to make
2757 * a forward progress or bypass the charge if the oom killer
2758 * couldn't make any progress.
2759 */
2760 if (mem_cgroup_oom(mem_over_limit, gfp_mask,
2761 get_order(nr_pages * PAGE_SIZE))) {
2762 passed_oom = true;
2763 nr_retries = MAX_RECLAIM_RETRIES;
2764 goto retry;
2765 }
2766nomem:
2767 /*
2768 * Memcg doesn't have a dedicated reserve for atomic
2769 * allocations. But like the global atomic pool, we need to
2770 * put the burden of reclaim on regular allocation requests
2771 * and let these go through as privileged allocations.
2772 */
2773 if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
2774 return -ENOMEM;
2775force:
2776 /*
2777 * If the allocation has to be enforced, don't forget to raise
2778 * a MEMCG_MAX event.
2779 */
2780 if (!raised_max_event)
2781 __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
2782
2783 /*
2784 * The allocation either can't fail or will lead to more memory
2785 * being freed very soon. Allow memory usage go over the limit
2786 * temporarily by force charging it.
2787 */
2788 page_counter_charge(&memcg->memory, nr_pages);
2789 if (do_memsw_account())
2790 page_counter_charge(&memcg->memsw, nr_pages);
2791
2792 return 0;
2793
2794done_restock:
2795 if (batch > nr_pages)
2796 refill_stock(memcg, batch - nr_pages);
2797
2798 /*
2799 * If the hierarchy is above the normal consumption range, schedule
2800 * reclaim on returning to userland. We can perform reclaim here
2801 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2802 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2803 * not recorded as it most likely matches current's and won't
2804 * change in the meantime. As high limit is checked again before
2805 * reclaim, the cost of mismatch is negligible.
2806 */
2807 do {
2808 bool mem_high, swap_high;
2809
2810 mem_high = page_counter_read(&memcg->memory) >
2811 READ_ONCE(memcg->memory.high);
2812 swap_high = page_counter_read(&memcg->swap) >
2813 READ_ONCE(memcg->swap.high);
2814
2815 /* Don't bother a random interrupted task */
2816 if (!in_task()) {
2817 if (mem_high) {
2818 schedule_work(&memcg->high_work);
2819 break;
2820 }
2821 continue;
2822 }
2823
2824 if (mem_high || swap_high) {
2825 /*
2826 * The allocating tasks in this cgroup will need to do
2827 * reclaim or be throttled to prevent further growth
2828 * of the memory or swap footprints.
2829 *
2830 * Target some best-effort fairness between the tasks,
2831 * and distribute reclaim work and delay penalties
2832 * based on how much each task is actually allocating.
2833 */
2834 current->memcg_nr_pages_over_high += batch;
2835 set_notify_resume(current);
2836 break;
2837 }
2838 } while ((memcg = parent_mem_cgroup(memcg)));
2839
2840 /*
2841 * Reclaim is set up above to be called from the userland
2842 * return path. But also attempt synchronous reclaim to avoid
2843 * excessive overrun while the task is still inside the
2844 * kernel. If this is successful, the return path will see it
2845 * when it rechecks the overage and simply bail out.
2846 */
2847 if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH &&
2848 !(current->flags & PF_MEMALLOC) &&
2849 gfpflags_allow_blocking(gfp_mask))
2850 __mem_cgroup_handle_over_high(gfp_mask);
2851 return 0;
2852}
2853
2854static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2855 unsigned int nr_pages)
2856{
2857 if (mem_cgroup_is_root(memcg))
2858 return 0;
2859
2860 return try_charge_memcg(memcg, gfp_mask, nr_pages);
2861}
2862
2863static void commit_charge(struct folio *folio, struct obj_cgroup *objcg)
2864{
2865 VM_BUG_ON_FOLIO(folio_memcg_charged(folio), folio);
2866 /*
2867 * Any of the following ensures folio's objcg stability:
2868 *
2869 * - the page lock
2870 * - LRU isolation
2871 * - exclusive reference
2872 */
2873 folio->memcg_data = (unsigned long)objcg;
2874}
2875
2876#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
2877static inline void account_slab_nmi_safe(struct mem_cgroup *memcg,
2878 struct pglist_data *pgdat,
2879 enum node_stat_item idx, int nr)
2880{
2881 struct lruvec *lruvec;
2882
2883 if (likely(!in_nmi())) {
2884 lruvec = mem_cgroup_lruvec(memcg, pgdat);
2885 mod_memcg_lruvec_state(lruvec, idx, nr);
2886 } else {
2887 struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
2888
2889 /* preemption is disabled in_nmi(). */
2890 __css_rstat_updated(&memcg->css, smp_processor_id());
2891 if (idx == NR_SLAB_RECLAIMABLE_B)
2892 atomic_add(nr, &pn->slab_reclaimable);
2893 else
2894 atomic_add(nr, &pn->slab_unreclaimable);
2895 }
2896}
2897#else
2898static inline void account_slab_nmi_safe(struct mem_cgroup *memcg,
2899 struct pglist_data *pgdat,
2900 enum node_stat_item idx, int nr)
2901{
2902 struct lruvec *lruvec;
2903
2904 lruvec = mem_cgroup_lruvec(memcg, pgdat);
2905 mod_memcg_lruvec_state(lruvec, idx, nr);
2906}
2907#endif
2908
2909static inline void mod_objcg_mlstate(struct obj_cgroup *objcg,
2910 struct pglist_data *pgdat,
2911 enum node_stat_item idx, int nr)
2912{
2913 struct mem_cgroup *memcg;
2914
2915 rcu_read_lock();
2916 memcg = obj_cgroup_memcg(objcg);
2917 account_slab_nmi_safe(memcg, pgdat, idx, nr);
2918 rcu_read_unlock();
2919}
2920
2921static __always_inline
2922struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)
2923{
2924 /*
2925 * Slab objects are accounted individually, not per-page.
2926 * Memcg membership data for each individual object is saved in
2927 * slab->obj_exts.
2928 */
2929 unsigned long obj_exts;
2930 struct slabobj_ext *obj_ext;
2931 struct obj_cgroup *objcg;
2932
2933 obj_exts = slab_obj_exts(slab);
2934 if (!obj_exts)
2935 return NULL;
2936
2937 if (!slab_needs_objcg(slab))
2938 return NULL;
2939
2940 get_slab_obj_exts(obj_exts);
2941 obj_ext = slab_obj_ext(slab->slab_cache, slab, obj_exts, p);
2942 objcg = slab_obj_ext_objcg(slab, obj_ext);
2943 if (objcg) {
2944 put_slab_obj_exts(obj_exts);
2945 return obj_cgroup_memcg(objcg);
2946 }
2947 put_slab_obj_exts(obj_exts);
2948
2949 return NULL;
2950}
2951
2952/*
2953 * Returns a pointer to the memory cgroup to which the kernel object is charged.
2954 * It is not suitable for objects allocated using vmalloc().
2955 *
2956 * A passed kernel object must be a slab object or a generic kernel page.
2957 *
2958 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2959 * cgroup_mutex, etc.
2960 */
2961struct mem_cgroup *mem_cgroup_from_virt(void *p)
2962{
2963 struct slab *slab;
2964
2965 if (mem_cgroup_disabled())
2966 return NULL;
2967
2968 slab = virt_to_slab(p);
2969 if (slab)
2970 return mem_cgroup_from_obj_slab(slab, p);
2971 return folio_memcg_check(virt_to_folio(p));
2972}
2973
2974static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg,
2975 int nid)
2976{
2977 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2978 struct obj_cgroup *objcg = rcu_dereference(memcg->nodeinfo[nid]->objcg);
2979
2980 if (likely(objcg && obj_cgroup_tryget(objcg)))
2981 return objcg;
2982 }
2983
2984 return NULL;
2985}
2986
2987static inline struct obj_cgroup *get_obj_cgroup_from_memcg(struct mem_cgroup *memcg,
2988 int nid)
2989{
2990 struct obj_cgroup *objcg;
2991
2992 rcu_read_lock();
2993 objcg = __get_obj_cgroup_from_memcg(memcg, nid);
2994 rcu_read_unlock();
2995
2996 return objcg;
2997}
2998
2999static struct obj_cgroup *current_objcg_update(void)
3000{
3001 struct mem_cgroup *memcg;
3002 struct obj_cgroup *old, *objcg = NULL;
3003
3004 do {
3005 /* Atomically drop the update bit. */
3006 old = xchg(&current->objcg, NULL);
3007 if (old) {
3008 old = (struct obj_cgroup *)
3009 ((unsigned long)old & ~CURRENT_OBJCG_UPDATE_FLAG);
3010 obj_cgroup_put(old);
3011
3012 old = NULL;
3013 }
3014
3015 /* If new objcg is NULL, no reason for the second atomic update. */
3016 if (!current->mm || (current->flags & PF_KTHREAD))
3017 return NULL;
3018
3019 /*
3020 * Release the objcg pointer from the previous iteration,
3021 * if try_cmpxcg() below fails.
3022 */
3023 if (unlikely(objcg)) {
3024 obj_cgroup_put(objcg);
3025 objcg = NULL;
3026 }
3027
3028 /*
3029 * Obtain the new objcg pointer. The current task can be
3030 * asynchronously moved to another memcg and the previous
3031 * memcg can be offlined. So let's get the memcg pointer
3032 * and try get a reference to objcg under a rcu read lock.
3033 */
3034
3035 rcu_read_lock();
3036 memcg = mem_cgroup_from_task(current);
3037 objcg = __get_obj_cgroup_from_memcg(memcg, numa_node_id());
3038 rcu_read_unlock();
3039
3040 /*
3041 * Try set up a new objcg pointer atomically. If it
3042 * fails, it means the update flag was set concurrently, so
3043 * the whole procedure should be repeated.
3044 */
3045 } while (!try_cmpxchg(&current->objcg, &old, objcg));
3046
3047 return objcg;
3048}
3049
3050__always_inline struct obj_cgroup *current_obj_cgroup(void)
3051{
3052 struct mem_cgroup *memcg;
3053 struct obj_cgroup *objcg;
3054 int nid = numa_node_id();
3055
3056 if (IS_ENABLED(CONFIG_MEMCG_NMI_UNSAFE) && in_nmi())
3057 return NULL;
3058
3059 if (in_task()) {
3060 memcg = current->active_memcg;
3061 if (unlikely(memcg))
3062 goto from_memcg;
3063
3064 objcg = READ_ONCE(current->objcg);
3065 if (unlikely((unsigned long)objcg & CURRENT_OBJCG_UPDATE_FLAG))
3066 objcg = current_objcg_update();
3067 /*
3068 * Objcg reference is kept by the task, so it's safe
3069 * to use the objcg by the current task.
3070 */
3071 return objcg ? : rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1);
3072 }
3073
3074 memcg = this_cpu_read(int_active_memcg);
3075 if (unlikely(memcg))
3076 goto from_memcg;
3077
3078 return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1);
3079
3080from_memcg:
3081 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
3082 /*
3083 * Memcg pointer is protected by scope (see set_active_memcg())
3084 * and is pinning the corresponding objcg, so objcg can't go
3085 * away and can be used within the scope without any additional
3086 * protection.
3087 */
3088 objcg = rcu_dereference_check(memcg->nodeinfo[nid]->objcg, 1);
3089 if (likely(objcg))
3090 return objcg;
3091 }
3092
3093 return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1);
3094}
3095
3096struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
3097{
3098 struct obj_cgroup *objcg;
3099
3100 objcg = folio_objcg(folio);
3101 if (objcg)
3102 obj_cgroup_get(objcg);
3103
3104 return objcg;
3105}
3106
3107#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
3108static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val)
3109{
3110 if (likely(!in_nmi())) {
3111 mod_memcg_state(memcg, MEMCG_KMEM, val);
3112 } else {
3113 /* preemption is disabled in_nmi(). */
3114 __css_rstat_updated(&memcg->css, smp_processor_id());
3115 atomic_add(val, &memcg->kmem_stat);
3116 }
3117}
3118#else
3119static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val)
3120{
3121 mod_memcg_state(memcg, MEMCG_KMEM, val);
3122}
3123#endif
3124
3125/*
3126 * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a objcg
3127 * @objcg: object cgroup to uncharge
3128 * @nr_pages: number of pages to uncharge
3129 */
3130static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
3131 unsigned int nr_pages)
3132{
3133 struct mem_cgroup *memcg;
3134
3135 memcg = get_mem_cgroup_from_objcg(objcg);
3136
3137 account_kmem_nmi_safe(memcg, -nr_pages);
3138 memcg1_account_kmem(memcg, -nr_pages);
3139 if (!mem_cgroup_is_root(memcg))
3140 refill_stock(memcg, nr_pages);
3141
3142 css_put(&memcg->css);
3143}
3144
3145/*
3146 * obj_cgroup_charge_pages: charge a number of kernel pages to a objcg
3147 * @objcg: object cgroup to charge
3148 * @gfp: reclaim mode
3149 * @nr_pages: number of pages to charge
3150 *
3151 * Returns 0 on success, an error code on failure.
3152 */
3153static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp,
3154 unsigned int nr_pages)
3155{
3156 struct mem_cgroup *memcg;
3157 int ret;
3158
3159 memcg = get_mem_cgroup_from_objcg(objcg);
3160
3161 ret = try_charge(memcg, gfp, nr_pages);
3162 if (ret)
3163 goto out;
3164
3165 account_kmem_nmi_safe(memcg, nr_pages);
3166 memcg1_account_kmem(memcg, nr_pages);
3167out:
3168 css_put(&memcg->css);
3169
3170 return ret;
3171}
3172
3173static struct obj_cgroup *page_objcg(const struct page *page)
3174{
3175 unsigned long memcg_data = page->memcg_data;
3176
3177 if (mem_cgroup_disabled() || !memcg_data)
3178 return NULL;
3179
3180 VM_BUG_ON_PAGE((memcg_data & OBJEXTS_FLAGS_MASK) != MEMCG_DATA_KMEM,
3181 page);
3182 return (struct obj_cgroup *)(memcg_data - MEMCG_DATA_KMEM);
3183}
3184
3185static void page_set_objcg(struct page *page, const struct obj_cgroup *objcg)
3186{
3187 page->memcg_data = (unsigned long)objcg | MEMCG_DATA_KMEM;
3188}
3189
3190/**
3191 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
3192 * @page: page to charge
3193 * @gfp: reclaim mode
3194 * @order: allocation order
3195 *
3196 * Returns 0 on success, an error code on failure.
3197 */
3198int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
3199{
3200 struct obj_cgroup *objcg;
3201 int ret = 0;
3202
3203 objcg = current_obj_cgroup();
3204 if (objcg && !obj_cgroup_is_root(objcg)) {
3205 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
3206 if (!ret) {
3207 obj_cgroup_get(objcg);
3208 page_set_objcg(page, objcg);
3209 return 0;
3210 }
3211 }
3212 return ret;
3213}
3214
3215/**
3216 * __memcg_kmem_uncharge_page: uncharge a kmem page
3217 * @page: page to uncharge
3218 * @order: allocation order
3219 */
3220void __memcg_kmem_uncharge_page(struct page *page, int order)
3221{
3222 struct obj_cgroup *objcg = page_objcg(page);
3223 unsigned int nr_pages = 1 << order;
3224
3225 if (!objcg)
3226 return;
3227
3228 obj_cgroup_uncharge_pages(objcg, nr_pages);
3229 page->memcg_data = 0;
3230 obj_cgroup_put(objcg);
3231}
3232
3233static struct obj_stock_pcp *trylock_stock(void)
3234{
3235 if (local_trylock(&obj_stock.lock))
3236 return this_cpu_ptr(&obj_stock);
3237
3238 return NULL;
3239}
3240
3241static void unlock_stock(struct obj_stock_pcp *stock)
3242{
3243 if (stock)
3244 local_unlock(&obj_stock.lock);
3245}
3246
3247/* Call after __refill_obj_stock() so a slot for objcg exists in the stock */
3248static void __account_obj_stock(struct obj_cgroup *objcg,
3249 struct obj_stock_pcp *stock, int nr,
3250 struct pglist_data *pgdat, enum node_stat_item idx)
3251{
3252 int16_t *bytes;
3253 int i;
3254
3255 /*
3256 * Though at the moment MAX_NUMNODES <= 1024 in all archs but let's make
3257 * sure it does not exceed S16_MAX otherwise we need to fix node_id type
3258 * in struct obj_stock_pcp.
3259 */
3260 BUILD_BUG_ON(MAX_NUMNODES >= S16_MAX);
3261
3262 if (!stock)
3263 goto direct;
3264
3265 for (i = 0; i < NR_OBJ_STOCK; ++i) {
3266 if (READ_ONCE(stock->cached[i]) == objcg)
3267 break;
3268 }
3269 if (i == NR_OBJ_STOCK)
3270 goto direct;
3271
3272 /*
3273 * Save vmstat data in stock and skip vmstat array update unless
3274 * accumulating over a page of vmstat data or when the objcg slot or
3275 * pgdat the stats belong to changes.
3276 */
3277 if (stock->index < 0) {
3278 stock->index = i;
3279 stock->node_id = pgdat->node_id;
3280 } else if (stock->index != i || stock->node_id != pgdat->node_id) {
3281 struct obj_cgroup *old = READ_ONCE(stock->cached[stock->index]);
3282 struct pglist_data *oldpg = NODE_DATA(stock->node_id);
3283
3284 if (stock->nr_slab_reclaimable_b) {
3285 mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B,
3286 stock->nr_slab_reclaimable_b);
3287 stock->nr_slab_reclaimable_b = 0;
3288 }
3289 if (stock->nr_slab_unreclaimable_b) {
3290 mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B,
3291 stock->nr_slab_unreclaimable_b);
3292 stock->nr_slab_unreclaimable_b = 0;
3293 }
3294 stock->index = i;
3295 stock->node_id = pgdat->node_id;
3296 }
3297
3298 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b
3299 : &stock->nr_slab_unreclaimable_b;
3300
3301 /*
3302 * Fold @nr into the cached value and decide whether to keep it cached
3303 * or flush it directly. Cache the combined value when it fits in the
3304 * int16_t storage and either the cache was empty (so even a value
3305 * above PAGE_SIZE gets a chance to be canceled by a paired delta) or
3306 * the combined value is within the PAGE_SIZE flush threshold.
3307 */
3308 nr += *bytes;
3309 if (abs(nr) <= S16_MAX && (!*bytes || abs(nr) <= PAGE_SIZE)) {
3310 *bytes = nr;
3311 nr = 0;
3312 } else {
3313 *bytes = 0;
3314 }
3315direct:
3316 if (nr)
3317 mod_objcg_mlstate(objcg, pgdat, idx, nr);
3318}
3319
3320static bool __consume_obj_stock(struct obj_cgroup *objcg,
3321 struct obj_stock_pcp *stock,
3322 unsigned int nr_bytes)
3323{
3324 int i;
3325
3326 for (i = 0; i < NR_OBJ_STOCK; ++i) {
3327 if (READ_ONCE(stock->cached[i]) != objcg)
3328 continue;
3329 if (stock->nr_bytes[i] >= nr_bytes) {
3330 stock->nr_bytes[i] -= nr_bytes;
3331 return true;
3332 }
3333 return false;
3334 }
3335
3336 return false;
3337}
3338
3339static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3340{
3341 struct obj_stock_pcp *stock;
3342 bool ret = false;
3343
3344 stock = trylock_stock();
3345 if (!stock)
3346 return ret;
3347
3348 ret = __consume_obj_stock(objcg, stock, nr_bytes);
3349 unlock_stock(stock);
3350
3351 return ret;
3352}
3353
3354/* Flush the cached slab stats (if any) back to their owning objcg/pgdat. */
3355static void drain_obj_stock_stats(struct obj_stock_pcp *stock)
3356{
3357 struct obj_cgroup *old;
3358 struct pglist_data *oldpg;
3359
3360 if (stock->index < 0)
3361 return;
3362
3363 old = READ_ONCE(stock->cached[stock->index]);
3364 oldpg = NODE_DATA(stock->node_id);
3365
3366 if (stock->nr_slab_reclaimable_b) {
3367 mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B,
3368 stock->nr_slab_reclaimable_b);
3369 stock->nr_slab_reclaimable_b = 0;
3370 }
3371 if (stock->nr_slab_unreclaimable_b) {
3372 mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B,
3373 stock->nr_slab_unreclaimable_b);
3374 stock->nr_slab_unreclaimable_b = 0;
3375 }
3376 stock->index = -1;
3377 stock->node_id = NUMA_NO_NODE;
3378}
3379
3380static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i)
3381{
3382 struct obj_cgroup *old = READ_ONCE(stock->cached[i]);
3383
3384 if (!old)
3385 return;
3386
3387 if (stock->nr_bytes[i]) {
3388 unsigned int nr_pages = stock->nr_bytes[i] >> PAGE_SHIFT;
3389 unsigned int nr_bytes = stock->nr_bytes[i] & (PAGE_SIZE - 1);
3390
3391 if (nr_pages) {
3392 struct mem_cgroup *memcg;
3393
3394 memcg = get_mem_cgroup_from_objcg(old);
3395
3396 memcg_uncharge_kmem(memcg, nr_pages);
3397
3398 css_put(&memcg->css);
3399 }
3400
3401 /*
3402 * The leftover is flushed to the centralized per-memcg value.
3403 * On the next attempt to refill obj stock it will be moved
3404 * to a per-cpu stock (probably, on an other CPU), see
3405 * refill_obj_stock().
3406 *
3407 * How often it's flushed is a trade-off between the memory
3408 * limit enforcement accuracy and potential CPU contention,
3409 * so it might be changed in the future.
3410 */
3411 atomic_add(nr_bytes, &old->nr_charged_bytes);
3412 stock->nr_bytes[i] = 0;
3413 }
3414
3415 /* Flush vmstat data when its owning slot is being drained. */
3416 if (stock->index == i)
3417 drain_obj_stock_stats(stock);
3418
3419 WRITE_ONCE(stock->cached[i], NULL);
3420 obj_cgroup_put(old);
3421}
3422
3423static void drain_obj_stock(struct obj_stock_pcp *stock)
3424{
3425 int i;
3426
3427 for (i = 0; i < NR_OBJ_STOCK; ++i)
3428 drain_obj_stock_slot(stock, i);
3429}
3430
3431static bool obj_stock_flush_required(struct obj_stock_pcp *stock,
3432 struct mem_cgroup *root_memcg)
3433{
3434 struct obj_cgroup *objcg;
3435 struct mem_cgroup *memcg;
3436 bool flush = false;
3437 int i;
3438
3439 rcu_read_lock();
3440 for (i = 0; i < NR_OBJ_STOCK; ++i) {
3441 objcg = READ_ONCE(stock->cached[i]);
3442 if (!objcg)
3443 continue;
3444 memcg = obj_cgroup_memcg(objcg);
3445 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) {
3446 flush = true;
3447 break;
3448 }
3449 }
3450 rcu_read_unlock();
3451
3452 return flush;
3453}
3454
3455static void __refill_obj_stock(struct obj_cgroup *objcg,
3456 struct obj_stock_pcp *stock,
3457 unsigned int nr_bytes,
3458 bool allow_uncharge)
3459{
3460 unsigned int nr_pages = 0;
3461 unsigned int stock_nr_bytes;
3462 int i, slot = -1, empty_slot = -1;
3463
3464 if (!stock) {
3465 nr_pages = nr_bytes >> PAGE_SHIFT;
3466 nr_bytes = nr_bytes & (PAGE_SIZE - 1);
3467 atomic_add(nr_bytes, &objcg->nr_charged_bytes);
3468 goto out;
3469 }
3470
3471 for (i = 0; i < NR_OBJ_STOCK; ++i) {
3472 struct obj_cgroup *cached = READ_ONCE(stock->cached[i]);
3473
3474 if (!cached) {
3475 if (empty_slot == -1)
3476 empty_slot = i;
3477 continue;
3478 }
3479 if (cached == objcg) {
3480 slot = i;
3481 break;
3482 }
3483 }
3484
3485 if (slot == -1) {
3486 slot = empty_slot;
3487 if (slot == -1) {
3488 slot = stock->drain_idx++;
3489 if (stock->drain_idx == NR_OBJ_STOCK)
3490 stock->drain_idx = 0;
3491 drain_obj_stock_slot(stock, slot);
3492 }
3493 obj_cgroup_get(objcg);
3494 /*
3495 * Keep the xchg result in the unsigned int local; storing
3496 * it directly into stock->nr_bytes[slot] (uint16_t) would
3497 * silently truncate values >= U16_MAX and bypass the flush
3498 * guard below, leaking page-counter charges.
3499 */
3500 stock_nr_bytes = atomic_read(&objcg->nr_charged_bytes)
3501 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0;
3502 WRITE_ONCE(stock->cached[slot], objcg);
3503
3504 allow_uncharge = true; /* Allow uncharge when objcg changes */
3505 } else {
3506 stock_nr_bytes = stock->nr_bytes[slot];
3507 }
3508
3509 stock_nr_bytes += nr_bytes;
3510
3511 if ((allow_uncharge && (stock_nr_bytes > PAGE_SIZE)) ||
3512 stock_nr_bytes > U16_MAX) {
3513 nr_pages = stock_nr_bytes >> PAGE_SHIFT;
3514 stock_nr_bytes &= (PAGE_SIZE - 1);
3515 }
3516 stock->nr_bytes[slot] = stock_nr_bytes;
3517
3518out:
3519 if (nr_pages)
3520 obj_cgroup_uncharge_pages(objcg, nr_pages);
3521}
3522
3523static void refill_obj_stock(struct obj_cgroup *objcg,
3524 unsigned int nr_bytes,
3525 bool allow_uncharge)
3526{
3527 struct obj_stock_pcp *stock = trylock_stock();
3528 __refill_obj_stock(objcg, stock, nr_bytes, allow_uncharge);
3529 unlock_stock(stock);
3530}
3531
3532static int __obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp,
3533 size_t size, size_t *remainder)
3534{
3535 size_t charge_size;
3536 int ret;
3537
3538 charge_size = PAGE_ALIGN(size);
3539 ret = obj_cgroup_charge_pages(objcg, gfp, charge_size >> PAGE_SHIFT);
3540 if (!ret)
3541 *remainder = charge_size - size;
3542
3543 return ret;
3544}
3545
3546int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3547{
3548 size_t remainder;
3549 int ret;
3550
3551 if (likely(consume_obj_stock(objcg, size)))
3552 return 0;
3553
3554 /*
3555 * In theory, objcg->nr_charged_bytes can have enough
3556 * pre-charged bytes to satisfy the allocation. However,
3557 * flushing objcg->nr_charged_bytes requires two atomic
3558 * operations, and objcg->nr_charged_bytes can't be big.
3559 * The shared objcg->nr_charged_bytes can also become a
3560 * performance bottleneck if all tasks of the same memcg are
3561 * trying to update it. So it's better to ignore it and try
3562 * grab some new pages. The stock's nr_bytes will be flushed to
3563 * objcg->nr_charged_bytes later on when objcg changes.
3564 *
3565 * The stock's nr_bytes may contain enough pre-charged bytes
3566 * to allow one less page from being charged, but we can't rely
3567 * on the pre-charged bytes not being changed outside of
3568 * consume_obj_stock() or refill_obj_stock(). So ignore those
3569 * pre-charged bytes as well when charging pages. To avoid a
3570 * page uncharge right after a page charge, we set the
3571 * allow_uncharge flag to false when calling refill_obj_stock()
3572 * to temporarily allow the pre-charged bytes to exceed the page
3573 * size limit. The maximum reachable value of the pre-charged
3574 * bytes is (sizeof(object) + PAGE_SIZE - 2) if there is no data
3575 * race.
3576 */
3577 ret = __obj_cgroup_charge(objcg, gfp, size, &remainder);
3578 if (!ret && remainder)
3579 refill_obj_stock(objcg, remainder, false);
3580
3581 return ret;
3582}
3583
3584void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3585{
3586 refill_obj_stock(objcg, size, true);
3587}
3588
3589static inline size_t obj_full_size(struct kmem_cache *s)
3590{
3591 /*
3592 * For each accounted object there is an extra space which is used
3593 * to store obj_cgroup membership. Charge it too.
3594 */
3595 return s->size + sizeof(struct obj_cgroup *);
3596}
3597
3598bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
3599 gfp_t flags, unsigned int slab_alloc_flags,
3600 size_t size, void **p)
3601{
3602 size_t obj_size = obj_full_size(s);
3603 struct obj_cgroup *objcg;
3604 struct slab *slab;
3605 size_t i;
3606
3607 /*
3608 * The obtained objcg pointer is safe to use within the current scope,
3609 * defined by current task or set_active_memcg() pair.
3610 * obj_cgroup_get() is used to get a permanent reference.
3611 */
3612 objcg = current_obj_cgroup();
3613 if (!objcg || obj_cgroup_is_root(objcg))
3614 return true;
3615
3616 /*
3617 * slab_alloc_node() avoids the NULL check, so we might be called with a
3618 * single NULL object. kmem_cache_alloc_bulk() aborts if it can't fill
3619 * the whole requested size.
3620 * return success as there's nothing to free back
3621 */
3622 if (unlikely(*p == NULL))
3623 return true;
3624
3625 flags &= gfp_allowed_mask;
3626
3627 if (lru) {
3628 int ret;
3629 struct mem_cgroup *memcg;
3630
3631 memcg = get_mem_cgroup_from_objcg(objcg);
3632 ret = memcg_list_lru_alloc(memcg, lru, flags);
3633 css_put(&memcg->css);
3634
3635 if (ret)
3636 return false;
3637 }
3638
3639 for (i = 0; i < size; i++) {
3640 unsigned long obj_exts;
3641 struct slabobj_ext *obj_ext;
3642 struct obj_stock_pcp *stock;
3643
3644 slab = virt_to_slab(p[i]);
3645
3646 if (!slab_obj_exts(slab)) {
3647 if (is_kfence_address(p[i]))
3648 continue;
3649 if (alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags))
3650 continue;
3651 }
3652
3653 /*
3654 * if we fail and size is 1, memcg_alloc_abort_single() will
3655 * just free the object, which is ok as we have not assigned
3656 * objcg to its obj_ext yet
3657 *
3658 * for larger sizes, kmem_cache_free_bulk() will uncharge
3659 * any objects that were already charged and obj_ext assigned
3660 *
3661 * TODO: we could batch this until slab_pgdat(slab) changes
3662 * between iterations, with a more complicated undo
3663 */
3664 stock = trylock_stock();
3665 if (!stock || !__consume_obj_stock(objcg, stock, obj_size)) {
3666 size_t remainder;
3667
3668 unlock_stock(stock);
3669 if (__obj_cgroup_charge(objcg, flags, obj_size, &remainder))
3670 return false;
3671 stock = trylock_stock();
3672 if (remainder)
3673 __refill_obj_stock(objcg, stock, remainder, false);
3674 }
3675 __account_obj_stock(objcg, stock, obj_size,
3676 slab_pgdat(slab), cache_vmstat_idx(s));
3677 unlock_stock(stock);
3678
3679 obj_exts = slab_obj_exts(slab);
3680 get_slab_obj_exts(obj_exts);
3681 obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]);
3682
3683 obj_cgroup_get(objcg);
3684 slab_obj_ext_set_objcg(slab, obj_ext, objcg);
3685
3686 put_slab_obj_exts(obj_exts);
3687 }
3688
3689 return true;
3690}
3691
3692void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
3693 void **p, int objects, unsigned long obj_exts)
3694{
3695 size_t obj_size = obj_full_size(s);
3696
3697 for (int i = 0; i < objects; i++) {
3698 struct obj_cgroup *objcg;
3699 struct slabobj_ext *obj_ext;
3700 struct obj_stock_pcp *stock;
3701
3702 obj_ext = slab_obj_ext(s, slab, obj_exts, p[i]);
3703 objcg = slab_obj_ext_objcg(slab, obj_ext);
3704 if (!objcg)
3705 continue;
3706
3707 slab_obj_ext_set_objcg(slab, obj_ext, NULL);
3708
3709 stock = trylock_stock();
3710 __refill_obj_stock(objcg, stock, obj_size, true);
3711 __account_obj_stock(objcg, stock, -obj_size,
3712 slab_pgdat(slab), cache_vmstat_idx(s));
3713 unlock_stock(stock);
3714
3715 obj_cgroup_put(objcg);
3716 }
3717}
3718
3719/*
3720 * The objcg is only set on the first page, so transfer it to all the
3721 * other pages.
3722 */
3723void split_page_memcg(struct page *page, unsigned order)
3724{
3725 struct obj_cgroup *objcg = page_objcg(page);
3726 unsigned int i, nr = 1 << order;
3727
3728 if (!objcg)
3729 return;
3730
3731 for (i = 1; i < nr; i++)
3732 page_set_objcg(&page[i], objcg);
3733
3734 obj_cgroup_get_many(objcg, nr - 1);
3735}
3736
3737void folio_split_memcg_refs(struct folio *folio, unsigned old_order,
3738 unsigned new_order)
3739{
3740 unsigned new_refs;
3741
3742 if (mem_cgroup_disabled() || !folio_memcg_charged(folio))
3743 return;
3744
3745 new_refs = (1 << (old_order - new_order)) - 1;
3746 obj_cgroup_get_many(folio_objcg(folio), new_refs);
3747}
3748
3749static void memcg_online_kmem(struct mem_cgroup *memcg)
3750{
3751 if (mem_cgroup_kmem_disabled())
3752 return;
3753
3754 if (unlikely(mem_cgroup_is_root(memcg)))
3755 return;
3756
3757 static_branch_enable(&memcg_kmem_online_key);
3758
3759 memcg->kmemcg_id = memcg->id.id;
3760}
3761
3762static void memcg_offline_kmem(struct mem_cgroup *memcg)
3763{
3764 struct mem_cgroup *parent;
3765
3766 if (mem_cgroup_kmem_disabled())
3767 return;
3768
3769 if (unlikely(mem_cgroup_is_root(memcg)))
3770 return;
3771
3772 parent = parent_mem_cgroup(memcg);
3773 memcg_reparent_list_lrus(memcg, parent);
3774}
3775
3776#ifdef CONFIG_CGROUP_WRITEBACK
3777
3778#include <trace/events/writeback.h>
3779
3780static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3781{
3782 return wb_domain_init(&memcg->cgwb_domain, gfp);
3783}
3784
3785static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3786{
3787 wb_domain_exit(&memcg->cgwb_domain);
3788}
3789
3790static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3791{
3792 wb_domain_size_changed(&memcg->cgwb_domain);
3793}
3794
3795struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3796{
3797 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3798
3799 if (!memcg->css.parent)
3800 return NULL;
3801
3802 return &memcg->cgwb_domain;
3803}
3804
3805/**
3806 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3807 * @wb: bdi_writeback in question
3808 * @pfilepages: out parameter for number of file pages
3809 * @pheadroom: out parameter for number of allocatable pages according to memcg
3810 * @pdirty: out parameter for number of dirty pages
3811 * @pwriteback: out parameter for number of pages under writeback
3812 *
3813 * Determine the numbers of file, headroom, dirty, and writeback pages in
3814 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
3815 * is a bit more involved.
3816 *
3817 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
3818 * headroom is calculated as the lowest headroom of itself and the
3819 * ancestors. Note that this doesn't consider the actual amount of
3820 * available memory in the system. The caller should further cap
3821 * *@pheadroom accordingly.
3822 */
3823void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3824 unsigned long *pheadroom, unsigned long *pdirty,
3825 unsigned long *pwriteback)
3826{
3827 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3828 struct mem_cgroup *parent;
3829
3830 mem_cgroup_flush_stats_ratelimited(memcg);
3831
3832 *pdirty = memcg_page_state(memcg, NR_FILE_DIRTY);
3833 *pwriteback = memcg_page_state(memcg, NR_WRITEBACK);
3834 *pfilepages = memcg_page_state(memcg, NR_INACTIVE_FILE) +
3835 memcg_page_state(memcg, NR_ACTIVE_FILE);
3836
3837 *pheadroom = PAGE_COUNTER_MAX;
3838 while ((parent = parent_mem_cgroup(memcg))) {
3839 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
3840 READ_ONCE(memcg->memory.high));
3841 unsigned long used = page_counter_read(&memcg->memory);
3842
3843 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3844 memcg = parent;
3845 }
3846}
3847
3848/*
3849 * Foreign dirty flushing
3850 *
3851 * There's an inherent mismatch between memcg and writeback. The former
3852 * tracks ownership per-page while the latter per-inode. This was a
3853 * deliberate design decision because honoring per-page ownership in the
3854 * writeback path is complicated, may lead to higher CPU and IO overheads
3855 * and deemed unnecessary given that write-sharing an inode across
3856 * different cgroups isn't a common use-case.
3857 *
3858 * Combined with inode majority-writer ownership switching, this works well
3859 * enough in most cases but there are some pathological cases. For
3860 * example, let's say there are two cgroups A and B which keep writing to
3861 * different but confined parts of the same inode. B owns the inode and
3862 * A's memory is limited far below B's. A's dirty ratio can rise enough to
3863 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
3864 * triggering background writeback. A will be slowed down without a way to
3865 * make writeback of the dirty pages happen.
3866 *
3867 * Conditions like the above can lead to a cgroup getting repeatedly and
3868 * severely throttled after making some progress after each
3869 * dirty_expire_interval while the underlying IO device is almost
3870 * completely idle.
3871 *
3872 * Solving this problem completely requires matching the ownership tracking
3873 * granularities between memcg and writeback in either direction. However,
3874 * the more egregious behaviors can be avoided by simply remembering the
3875 * most recent foreign dirtying events and initiating remote flushes on
3876 * them when local writeback isn't enough to keep the memory clean enough.
3877 *
3878 * The following two functions implement such mechanism. When a foreign
3879 * page - a page whose memcg and writeback ownerships don't match - is
3880 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
3881 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
3882 * decides that the memcg needs to sleep due to high dirty ratio, it calls
3883 * mem_cgroup_flush_foreign() which queues writeback on the recorded
3884 * foreign bdi_writebacks which haven't expired. Both the numbers of
3885 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
3886 * limited to MEMCG_CGWB_FRN_CNT.
3887 *
3888 * The mechanism only remembers IDs and doesn't hold any object references.
3889 * As being wrong occasionally doesn't matter, updates and accesses to the
3890 * records are lockless and racy.
3891 */
3892void mem_cgroup_track_foreign_dirty_slowpath(struct folio *folio,
3893 struct bdi_writeback *wb)
3894{
3895 struct mem_cgroup *memcg = folio_memcg(folio);
3896 struct memcg_cgwb_frn *frn;
3897 u64 now = get_jiffies_64();
3898 u64 oldest_at = now;
3899 int oldest = -1;
3900 int i;
3901
3902 trace_track_foreign_dirty(folio, wb);
3903
3904 /*
3905 * Pick the slot to use. If there is already a slot for @wb, keep
3906 * using it. If not replace the oldest one which isn't being
3907 * written out.
3908 */
3909 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
3910 frn = &memcg->cgwb_frn[i];
3911 if (frn->bdi_id == wb->bdi->id &&
3912 frn->memcg_id == wb->memcg_css->id)
3913 break;
3914 if (time_before64(frn->at, oldest_at) &&
3915 atomic_read(&frn->done.cnt) == 1) {
3916 oldest = i;
3917 oldest_at = frn->at;
3918 }
3919 }
3920
3921 if (i < MEMCG_CGWB_FRN_CNT) {
3922 /*
3923 * Re-using an existing one. Update timestamp lazily to
3924 * avoid making the cacheline hot. We want them to be
3925 * reasonably up-to-date and significantly shorter than
3926 * dirty_expire_interval as that's what expires the record.
3927 * Use the shorter of 1s and dirty_expire_interval / 8.
3928 */
3929 unsigned long update_intv =
3930 min_t(unsigned long, HZ,
3931 msecs_to_jiffies(dirty_expire_interval * 10) / 8);
3932
3933 if (time_before64(frn->at, now - update_intv))
3934 frn->at = now;
3935 } else if (oldest >= 0) {
3936 /* replace the oldest free one */
3937 frn = &memcg->cgwb_frn[oldest];
3938 frn->bdi_id = wb->bdi->id;
3939 frn->memcg_id = wb->memcg_css->id;
3940 frn->at = now;
3941 }
3942}
3943
3944/* issue foreign writeback flushes for recorded foreign dirtying events */
3945void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
3946{
3947 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3948 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
3949 u64 now = jiffies_64;
3950 int i;
3951
3952 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
3953 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
3954
3955 /*
3956 * If the record is older than dirty_expire_interval,
3957 * writeback on it has already started. No need to kick it
3958 * off again. Also, don't start a new one if there's
3959 * already one in flight.
3960 */
3961 if (time_after64(frn->at, now - intv) &&
3962 atomic_read(&frn->done.cnt) == 1) {
3963 frn->at = 0;
3964 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
3965 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id,
3966 WB_REASON_FOREIGN_FLUSH,
3967 &frn->done);
3968 }
3969 }
3970}
3971
3972#else /* CONFIG_CGROUP_WRITEBACK */
3973
3974static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3975{
3976 return 0;
3977}
3978
3979static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3980{
3981}
3982
3983static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3984{
3985}
3986
3987#endif /* CONFIG_CGROUP_WRITEBACK */
3988
3989/*
3990 * Private memory cgroup IDR
3991 *
3992 * Swap-out records and page cache shadow entries need to store memcg
3993 * references in constrained space, so we maintain an ID space that is
3994 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
3995 * memory-controlled cgroups to 64k.
3996 *
3997 * However, there usually are many references to the offline CSS after
3998 * the cgroup has been destroyed, such as page cache or reclaimable
3999 * slab objects, that don't need to hang on to the ID. We want to keep
4000 * those dead CSS from occupying IDs, or we might quickly exhaust the
4001 * relatively small ID space and prevent the creation of new cgroups
4002 * even when there are much fewer than 64k cgroups - possibly none.
4003 *
4004 * Maintain a private 16-bit ID space for memcg, and allow the ID to
4005 * be freed and recycled when it's no longer needed, which is usually
4006 * when the CSS is offlined.
4007 *
4008 * The only exception to that are records of swapped out tmpfs/shmem
4009 * pages that need to be attributed to live ancestors on swapin. But
4010 * those references are manageable from userspace.
4011 */
4012
4013#define MEM_CGROUP_ID_MAX ((1UL << MEM_CGROUP_ID_SHIFT) - 1)
4014static DEFINE_XARRAY_ALLOC1(mem_cgroup_private_ids);
4015
4016static void mem_cgroup_private_id_remove(struct mem_cgroup *memcg)
4017{
4018 if (memcg->id.id > 0) {
4019 xa_erase(&mem_cgroup_private_ids, memcg->id.id);
4020 memcg->id.id = 0;
4021 }
4022}
4023
4024static inline void mem_cgroup_private_id_put(struct mem_cgroup *memcg, unsigned int n)
4025{
4026 if (refcount_sub_and_test(n, &memcg->id.ref)) {
4027 mem_cgroup_private_id_remove(memcg);
4028
4029 /* Memcg ID pins CSS */
4030 css_put(&memcg->css);
4031 }
4032}
4033
4034struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, unsigned int n)
4035{
4036 while (!refcount_add_not_zero(n, &memcg->id.ref)) {
4037 /*
4038 * The root cgroup cannot be destroyed, so it's refcount must
4039 * always be >= 1.
4040 */
4041 if (WARN_ON_ONCE(mem_cgroup_is_root(memcg))) {
4042 VM_BUG_ON(1);
4043 break;
4044 }
4045 memcg = parent_mem_cgroup(memcg);
4046 }
4047 return memcg;
4048}
4049
4050/**
4051 * mem_cgroup_from_private_id - look up a memcg from a memcg id
4052 * @id: the memcg id to look up
4053 *
4054 * Caller must hold rcu_read_lock().
4055 */
4056struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id)
4057{
4058 WARN_ON_ONCE(!rcu_read_lock_held());
4059 return xa_load(&mem_cgroup_private_ids, id);
4060}
4061
4062struct mem_cgroup *mem_cgroup_get_from_id(u64 id)
4063{
4064 struct cgroup *cgrp;
4065 struct cgroup_subsys_state *css;
4066 struct mem_cgroup *memcg = NULL;
4067
4068 cgrp = cgroup_get_from_id(id);
4069 if (IS_ERR(cgrp))
4070 return NULL;
4071
4072 css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys);
4073 if (css)
4074 memcg = container_of(css, struct mem_cgroup, css);
4075
4076 cgroup_put(cgrp);
4077
4078 return memcg;
4079}
4080
4081static void free_mem_cgroup_per_node_info(struct mem_cgroup_per_node *pn)
4082{
4083 if (!pn)
4084 return;
4085
4086 free_percpu(pn->lruvec_stats_percpu);
4087 kfree(pn->lruvec_stats);
4088 kfree(pn);
4089}
4090
4091static bool alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4092{
4093 struct mem_cgroup_per_node *pn;
4094
4095 pn = kmem_cache_alloc_node(memcg_pn_cachep, GFP_KERNEL | __GFP_ZERO,
4096 node);
4097 if (!pn)
4098 return false;
4099
4100 pn->lruvec_stats = kzalloc_node(sizeof(struct lruvec_stats),
4101 GFP_KERNEL_ACCOUNT, node);
4102 if (!pn->lruvec_stats)
4103 goto fail;
4104
4105 pn->lruvec_stats_percpu = alloc_percpu_gfp(struct lruvec_stats_percpu,
4106 GFP_KERNEL_ACCOUNT);
4107 if (!pn->lruvec_stats_percpu)
4108 goto fail;
4109
4110 INIT_LIST_HEAD(&pn->objcg_list);
4111
4112 lruvec_init(&pn->lruvec);
4113 pn->memcg = memcg;
4114
4115 memcg->nodeinfo[node] = pn;
4116 return true;
4117fail:
4118 free_mem_cgroup_per_node_info(pn);
4119 return false;
4120}
4121
4122static void __mem_cgroup_free(struct mem_cgroup *memcg)
4123{
4124 int node;
4125
4126 for_each_node(node) {
4127 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
4128 if (!pn)
4129 continue;
4130
4131 obj_cgroup_put(pn->orig_objcg);
4132 free_mem_cgroup_per_node_info(pn);
4133 }
4134 memcg1_free_events(memcg);
4135 kfree(memcg->vmstats);
4136 free_percpu(memcg->vmstats_percpu);
4137 kfree(memcg);
4138}
4139
4140static void mem_cgroup_free(struct mem_cgroup *memcg)
4141{
4142 lru_gen_exit_memcg(memcg);
4143 memcg_wb_domain_exit(memcg);
4144 __mem_cgroup_free(memcg);
4145}
4146
4147static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
4148{
4149 struct memcg_vmstats_percpu *statc;
4150 struct memcg_vmstats_percpu __percpu *pstatc_pcpu;
4151 struct mem_cgroup *memcg;
4152 int node, cpu;
4153 int __maybe_unused i;
4154 long error;
4155
4156 memcg = kmem_cache_zalloc(memcg_cachep, GFP_KERNEL);
4157 if (!memcg)
4158 return ERR_PTR(-ENOMEM);
4159
4160 error = xa_alloc(&mem_cgroup_private_ids, &memcg->id.id, NULL,
4161 XA_LIMIT(1, MEM_CGROUP_ID_MAX), GFP_KERNEL);
4162 if (error)
4163 goto fail;
4164 error = -ENOMEM;
4165
4166 memcg->vmstats = kzalloc_obj(struct memcg_vmstats, GFP_KERNEL_ACCOUNT);
4167 if (!memcg->vmstats)
4168 goto fail;
4169
4170 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
4171 GFP_KERNEL_ACCOUNT);
4172 if (!memcg->vmstats_percpu)
4173 goto fail;
4174
4175 if (!memcg1_alloc_events(memcg))
4176 goto fail;
4177
4178 pstatc_pcpu = parent ? parent->vmstats_percpu : NULL;
4179 for_each_possible_cpu(cpu) {
4180 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
4181 statc->parent_pcpu = pstatc_pcpu;
4182 statc->vmstats = memcg->vmstats;
4183 }
4184
4185 for_each_node(node)
4186 if (!alloc_mem_cgroup_per_node_info(memcg, node))
4187 goto fail;
4188
4189 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4190 goto fail;
4191
4192 INIT_WORK(&memcg->high_work, high_work_func);
4193 vmpressure_init(&memcg->vmpressure);
4194 INIT_LIST_HEAD(&memcg->memory_peaks);
4195 INIT_LIST_HEAD(&memcg->swap_peaks);
4196 spin_lock_init(&memcg->peaks_lock);
4197 memcg->socket_pressure = get_jiffies_64();
4198#if BITS_PER_LONG < 64
4199 seqlock_init(&memcg->socket_pressure_seqlock);
4200#endif
4201 memcg1_memcg_init(memcg);
4202 memcg->kmemcg_id = -1;
4203#ifdef CONFIG_CGROUP_WRITEBACK
4204 INIT_LIST_HEAD(&memcg->cgwb_list);
4205 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
4206 memcg->cgwb_frn[i].done =
4207 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
4208#endif
4209 lru_gen_init_memcg(memcg);
4210 return memcg;
4211fail:
4212 mem_cgroup_private_id_remove(memcg);
4213 __mem_cgroup_free(memcg);
4214 return ERR_PTR(error);
4215}
4216
4217static struct cgroup_subsys_state * __ref
4218mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4219{
4220 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4221 struct mem_cgroup *memcg, *old_memcg;
4222 bool memcg_on_dfl = cgroup_subsys_on_dfl(memory_cgrp_subsys);
4223
4224 old_memcg = set_active_memcg(parent);
4225 memcg = mem_cgroup_alloc(parent);
4226 set_active_memcg(old_memcg);
4227 if (IS_ERR(memcg))
4228 return ERR_CAST(memcg);
4229
4230 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
4231 memcg1_soft_limit_reset(memcg);
4232#ifdef CONFIG_ZSWAP
4233 memcg->zswap_max = PAGE_COUNTER_MAX;
4234 WRITE_ONCE(memcg->zswap_writeback, true);
4235#endif
4236 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
4237 if (parent) {
4238 page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
4239 page_counter_init(&memcg->swap, &parent->swap, false);
4240#ifdef CONFIG_MEMCG_V1
4241 WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
4242 memcg->memory.track_failcnt = !memcg_on_dfl;
4243 memcg->memsw.track_failcnt = !memcg_on_dfl;
4244 WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
4245 page_counter_init(&memcg->kmem, &parent->kmem, false);
4246 page_counter_init(&memcg->tcpmem, &parent->tcpmem, false);
4247 memcg->tcpmem.track_failcnt = !memcg_on_dfl;
4248#endif
4249 } else {
4250 init_memcg_stats();
4251 init_memcg_events();
4252 page_counter_init(&memcg->memory, NULL, true);
4253 page_counter_init(&memcg->swap, NULL, false);
4254#ifdef CONFIG_MEMCG_V1
4255 page_counter_init(&memcg->kmem, NULL, false);
4256 page_counter_init(&memcg->tcpmem, NULL, false);
4257#endif
4258 root_mem_cgroup = memcg;
4259 return &memcg->css;
4260 }
4261
4262 if (memcg_on_dfl && !cgroup_memory_nosocket)
4263 static_branch_inc(&memcg_sockets_enabled_key);
4264
4265 if (!cgroup_memory_nobpf)
4266 static_branch_inc(&memcg_bpf_enabled_key);
4267
4268 return &memcg->css;
4269}
4270
4271static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4272{
4273 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4274 struct obj_cgroup *objcg;
4275 int nid;
4276
4277 memcg_online_kmem(memcg);
4278
4279 /*
4280 * A memcg must be visible for expand_shrinker_info()
4281 * by the time the maps are allocated. So, we allocate maps
4282 * here, when mem_cgroup_iter() can't skip it.
4283 */
4284 if (alloc_shrinker_info(memcg))
4285 goto offline_kmem;
4286
4287 for_each_node(nid) {
4288 objcg = obj_cgroup_alloc();
4289 if (!objcg)
4290 goto free_objcg;
4291
4292 if (unlikely(mem_cgroup_is_root(memcg)))
4293 objcg->is_root = true;
4294
4295 objcg->memcg = memcg;
4296 rcu_assign_pointer(memcg->nodeinfo[nid]->objcg, objcg);
4297 obj_cgroup_get(objcg);
4298 memcg->nodeinfo[nid]->orig_objcg = objcg;
4299 }
4300
4301 if (unlikely(mem_cgroup_is_root(memcg)) && !mem_cgroup_disabled())
4302 queue_delayed_work(system_dfl_wq, &stats_flush_dwork,
4303 FLUSH_TIME);
4304 lru_gen_online_memcg(memcg);
4305
4306 /* Online state pins memcg ID, memcg ID pins CSS */
4307 refcount_set(&memcg->id.ref, 1);
4308 css_get(css);
4309
4310 /*
4311 * Ensure mem_cgroup_from_private_id() works once we're fully online.
4312 *
4313 * We could do this earlier and require callers to filter with
4314 * css_tryget_online(). But right now there are no users that
4315 * need earlier access, and the workingset code relies on the
4316 * cgroup tree linkage (mem_cgroup_get_nr_swap_pages()). So
4317 * publish it here at the end of onlining. This matches the
4318 * regular ID destruction during offlining.
4319 */
4320 xa_store(&mem_cgroup_private_ids, memcg->id.id, memcg, GFP_KERNEL);
4321
4322 return 0;
4323free_objcg:
4324 for_each_node(nid) {
4325 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
4326
4327 objcg = rcu_replace_pointer(pn->objcg, NULL, true);
4328 if (objcg)
4329 percpu_ref_kill(&objcg->refcnt);
4330
4331 if (pn->orig_objcg) {
4332 obj_cgroup_put(pn->orig_objcg);
4333 /*
4334 * Reset pn->orig_objcg to NULL to prevent
4335 * obj_cgroup_put() from being called again in
4336 * __mem_cgroup_free().
4337 */
4338 pn->orig_objcg = NULL;
4339 }
4340 }
4341 free_shrinker_info(memcg);
4342offline_kmem:
4343 memcg_offline_kmem(memcg);
4344 mem_cgroup_private_id_remove(memcg);
4345 return -ENOMEM;
4346}
4347
4348static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4349{
4350 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4351
4352 memcg1_css_offline(memcg);
4353
4354 page_counter_set_min(&memcg->memory, 0);
4355 page_counter_set_low(&memcg->memory, 0);
4356
4357 zswap_memcg_offline_cleanup(memcg);
4358
4359 memcg_offline_kmem(memcg);
4360 /*
4361 * The reparenting of objcg must be after the reparenting of
4362 * the list_lru in memcg_offline_kmem(), which ensures that
4363 * they will not mistakenly get the parent list_lru.
4364 */
4365 memcg_reparent_objcgs(memcg);
4366 reparent_shrinker_deferred(memcg);
4367 wb_memcg_offline(memcg);
4368 lru_gen_offline_memcg(memcg);
4369
4370 drain_all_stock(memcg);
4371
4372 mem_cgroup_private_id_put(memcg, 1);
4373}
4374
4375static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4376{
4377 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4378
4379 invalidate_reclaim_iterators(memcg);
4380 lru_gen_release_memcg(memcg);
4381}
4382
4383static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4384{
4385 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4386 int __maybe_unused i;
4387
4388#ifdef CONFIG_CGROUP_WRITEBACK
4389 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
4390 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
4391#endif
4392 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4393 static_branch_dec(&memcg_sockets_enabled_key);
4394
4395 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg1_tcpmem_active(memcg))
4396 static_branch_dec(&memcg_sockets_enabled_key);
4397
4398 if (!cgroup_memory_nobpf)
4399 static_branch_dec(&memcg_bpf_enabled_key);
4400
4401 vmpressure_cleanup(&memcg->vmpressure);
4402 cancel_work_sync(&memcg->high_work);
4403 memcg1_remove_from_trees(memcg);
4404 free_shrinker_info(memcg);
4405 mem_cgroup_free(memcg);
4406}
4407
4408/**
4409 * mem_cgroup_css_reset - reset the states of a mem_cgroup
4410 * @css: the target css
4411 *
4412 * Reset the states of the mem_cgroup associated with @css. This is
4413 * invoked when the userland requests disabling on the default hierarchy
4414 * but the memcg is pinned through dependency. The memcg should stop
4415 * applying policies and should revert to the vanilla state as it may be
4416 * made visible again.
4417 *
4418 * The current implementation only resets the essential configurations.
4419 * This needs to be expanded to cover all the visible parts.
4420 */
4421static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4422{
4423 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4424
4425 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
4426 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
4427 WRITE_ONCE(memcg->oom_group, false);
4428#ifdef CONFIG_ZSWAP
4429 WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX);
4430 WRITE_ONCE(memcg->zswap_writeback, true);
4431#endif
4432#ifdef CONFIG_MEMCG_V1
4433 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
4434 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
4435#endif
4436 page_counter_set_min(&memcg->memory, 0);
4437 page_counter_set_low(&memcg->memory, 0);
4438 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
4439 memcg1_soft_limit_reset(memcg);
4440 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
4441 memcg_wb_domain_size_changed(memcg);
4442}
4443
4444struct aggregate_control {
4445 /* pointer to the aggregated (CPU and subtree aggregated) counters */
4446 long *aggregate;
4447 /* pointer to the non-hierarchichal (CPU aggregated) counters */
4448 long *local;
4449 /* pointer to the pending child counters during tree propagation */
4450 long *pending;
4451 /* pointer to the parent's pending counters, could be NULL */
4452 long *ppending;
4453 /* pointer to the percpu counters to be aggregated */
4454 long *cstat;
4455 /* pointer to the percpu counters of the last aggregation*/
4456 long *cstat_prev;
4457 /* size of the above counters */
4458 int size;
4459};
4460
4461static void mem_cgroup_stat_aggregate(struct aggregate_control *ac)
4462{
4463 int i;
4464 long delta, delta_cpu, v;
4465
4466 for (i = 0; i < ac->size; i++) {
4467 /*
4468 * Collect the aggregated propagation counts of groups
4469 * below us. We're in a per-cpu loop here and this is
4470 * a global counter, so the first cycle will get them.
4471 */
4472 delta = ac->pending[i];
4473 if (delta)
4474 ac->pending[i] = 0;
4475
4476 /* Add CPU changes on this level since the last flush */
4477 delta_cpu = 0;
4478 v = READ_ONCE(ac->cstat[i]);
4479 if (v != ac->cstat_prev[i]) {
4480 delta_cpu = v - ac->cstat_prev[i];
4481 delta += delta_cpu;
4482 ac->cstat_prev[i] = v;
4483 }
4484
4485 /* Aggregate counts on this level and propagate upwards */
4486 if (delta_cpu)
4487 ac->local[i] += delta_cpu;
4488
4489 if (delta) {
4490 ac->aggregate[i] += delta;
4491 if (ac->ppending)
4492 ac->ppending[i] += delta;
4493 }
4494 }
4495}
4496
4497#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
4498static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
4499{
4500 int nid;
4501
4502 if (atomic_read(&memcg->kmem_stat)) {
4503 int kmem = atomic_xchg(&memcg->kmem_stat, 0);
4504 int index = memcg_stats_index(MEMCG_KMEM);
4505
4506 memcg->vmstats->state[index] += kmem;
4507 memcg->vmstats->state_local[index] += kmem;
4508 if (parent)
4509 parent->vmstats->state_pending[index] += kmem;
4510 }
4511
4512 for_each_node_state(nid, N_MEMORY) {
4513 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
4514 struct lruvec_stats *lstats = pn->lruvec_stats;
4515 struct lruvec_stats *plstats = NULL;
4516
4517 if (parent)
4518 plstats = parent->nodeinfo[nid]->lruvec_stats;
4519
4520 if (atomic_read(&pn->slab_reclaimable)) {
4521 int slab = atomic_xchg(&pn->slab_reclaimable, 0);
4522 int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B);
4523
4524 lstats->state[index] += slab;
4525 lstats->state_local[index] += slab;
4526 if (plstats)
4527 plstats->state_pending[index] += slab;
4528 memcg->vmstats->state[index] += slab;
4529 memcg->vmstats->state_local[index] += slab;
4530 if (parent)
4531 parent->vmstats->state_pending[index] += slab;
4532 }
4533 if (atomic_read(&pn->slab_unreclaimable)) {
4534 int slab = atomic_xchg(&pn->slab_unreclaimable, 0);
4535 int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B);
4536
4537 lstats->state[index] += slab;
4538 lstats->state_local[index] += slab;
4539 if (plstats)
4540 plstats->state_pending[index] += slab;
4541 memcg->vmstats->state[index] += slab;
4542 memcg->vmstats->state_local[index] += slab;
4543 if (parent)
4544 parent->vmstats->state_pending[index] += slab;
4545 }
4546 }
4547}
4548#else
4549static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
4550{}
4551#endif
4552
4553static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
4554{
4555 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4556 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
4557 struct memcg_vmstats_percpu *statc;
4558 struct aggregate_control ac;
4559 int nid;
4560
4561 flush_nmi_stats(memcg, parent);
4562
4563 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
4564
4565 ac = (struct aggregate_control) {
4566 .aggregate = memcg->vmstats->state,
4567 .local = memcg->vmstats->state_local,
4568 .pending = memcg->vmstats->state_pending,
4569 .ppending = parent ? parent->vmstats->state_pending : NULL,
4570 .cstat = statc->state,
4571 .cstat_prev = statc->state_prev,
4572 .size = MEMCG_VMSTAT_SIZE,
4573 };
4574 mem_cgroup_stat_aggregate(&ac);
4575
4576 ac = (struct aggregate_control) {
4577 .aggregate = memcg->vmstats->events,
4578 .local = memcg->vmstats->events_local,
4579 .pending = memcg->vmstats->events_pending,
4580 .ppending = parent ? parent->vmstats->events_pending : NULL,
4581 .cstat = statc->events,
4582 .cstat_prev = statc->events_prev,
4583 .size = NR_MEMCG_EVENTS,
4584 };
4585 mem_cgroup_stat_aggregate(&ac);
4586
4587 for_each_node_state(nid, N_MEMORY) {
4588 struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid];
4589 struct lruvec_stats *lstats = pn->lruvec_stats;
4590 struct lruvec_stats *plstats = NULL;
4591 struct lruvec_stats_percpu *lstatc;
4592
4593 if (parent)
4594 plstats = parent->nodeinfo[nid]->lruvec_stats;
4595
4596 lstatc = per_cpu_ptr(pn->lruvec_stats_percpu, cpu);
4597
4598 ac = (struct aggregate_control) {
4599 .aggregate = lstats->state,
4600 .local = lstats->state_local,
4601 .pending = lstats->state_pending,
4602 .ppending = plstats ? plstats->state_pending : NULL,
4603 .cstat = lstatc->state,
4604 .cstat_prev = lstatc->state_prev,
4605 .size = NR_MEMCG_NODE_STAT_ITEMS,
4606 };
4607 mem_cgroup_stat_aggregate(&ac);
4608
4609 }
4610 WRITE_ONCE(statc->stats_updates, 0);
4611 /* We are in a per-cpu loop here, only do the atomic write once */
4612 if (atomic_long_read(&memcg->vmstats->stats_updates))
4613 atomic_long_set(&memcg->vmstats->stats_updates, 0);
4614}
4615
4616static void mem_cgroup_fork(struct task_struct *task)
4617{
4618 /*
4619 * Set the update flag to cause task->objcg to be initialized lazily
4620 * on the first allocation. It can be done without any synchronization
4621 * because it's always performed on the current task, so does
4622 * current_objcg_update().
4623 */
4624 task->objcg = (struct obj_cgroup *)CURRENT_OBJCG_UPDATE_FLAG;
4625}
4626
4627static void mem_cgroup_exit(struct task_struct *task)
4628{
4629 struct obj_cgroup *objcg = task->objcg;
4630
4631 objcg = (struct obj_cgroup *)
4632 ((unsigned long)objcg & ~CURRENT_OBJCG_UPDATE_FLAG);
4633 obj_cgroup_put(objcg);
4634
4635 /*
4636 * Some kernel allocations can happen after this point,
4637 * but let's ignore them. It can be done without any synchronization
4638 * because it's always performed on the current task, so does
4639 * current_objcg_update().
4640 */
4641 task->objcg = NULL;
4642}
4643
4644#ifdef CONFIG_LRU_GEN
4645static void mem_cgroup_lru_gen_attach(struct cgroup_taskset *tset)
4646{
4647 struct task_struct *task;
4648 struct cgroup_subsys_state *css;
4649
4650 /* find the first leader if there is any */
4651 cgroup_taskset_for_each_leader(task, css, tset)
4652 break;
4653
4654 if (!task)
4655 return;
4656
4657 task_lock(task);
4658 if (task->mm && READ_ONCE(task->mm->owner) == task)
4659 lru_gen_migrate_mm(task->mm);
4660 task_unlock(task);
4661}
4662#else
4663static void mem_cgroup_lru_gen_attach(struct cgroup_taskset *tset) {}
4664#endif /* CONFIG_LRU_GEN */
4665
4666static void mem_cgroup_kmem_attach(struct cgroup_taskset *tset)
4667{
4668 struct task_struct *task;
4669 struct cgroup_subsys_state *css;
4670
4671 cgroup_taskset_for_each(task, css, tset) {
4672 /* atomically set the update bit */
4673 set_bit(CURRENT_OBJCG_UPDATE_BIT, (unsigned long *)&task->objcg);
4674 }
4675}
4676
4677static void mem_cgroup_attach(struct cgroup_taskset *tset)
4678{
4679 mem_cgroup_lru_gen_attach(tset);
4680 mem_cgroup_kmem_attach(tset);
4681}
4682
4683static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
4684{
4685 if (value == PAGE_COUNTER_MAX)
4686 seq_puts(m, "max\n");
4687 else
4688 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
4689
4690 return 0;
4691}
4692
4693static u64 memory_current_read(struct cgroup_subsys_state *css,
4694 struct cftype *cft)
4695{
4696 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4697
4698 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
4699}
4700
4701#define OFP_PEAK_UNSET (((-1UL)))
4702
4703static int peak_show(struct seq_file *sf, void *v, struct page_counter *pc)
4704{
4705 struct cgroup_of_peak *ofp = of_peak(sf->private);
4706 u64 fd_peak = READ_ONCE(ofp->value), peak;
4707
4708 /* User wants global or local peak? */
4709 if (fd_peak == OFP_PEAK_UNSET)
4710 peak = pc->watermark;
4711 else
4712 peak = max(fd_peak, READ_ONCE(pc->local_watermark));
4713
4714 seq_printf(sf, "%llu\n", peak * PAGE_SIZE);
4715 return 0;
4716}
4717
4718static int memory_peak_show(struct seq_file *sf, void *v)
4719{
4720 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
4721
4722 return peak_show(sf, v, &memcg->memory);
4723}
4724
4725static int peak_open(struct kernfs_open_file *of)
4726{
4727 struct cgroup_of_peak *ofp = of_peak(of);
4728
4729 ofp->value = OFP_PEAK_UNSET;
4730 return 0;
4731}
4732
4733static void peak_release(struct kernfs_open_file *of)
4734{
4735 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4736 struct cgroup_of_peak *ofp = of_peak(of);
4737
4738 if (ofp->value == OFP_PEAK_UNSET) {
4739 /* fast path (no writes on this fd) */
4740 return;
4741 }
4742 spin_lock(&memcg->peaks_lock);
4743 list_del(&ofp->list);
4744 spin_unlock(&memcg->peaks_lock);
4745}
4746
4747static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes,
4748 loff_t off, struct page_counter *pc,
4749 struct list_head *watchers)
4750{
4751 unsigned long usage;
4752 struct cgroup_of_peak *peer_ctx;
4753 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4754 struct cgroup_of_peak *ofp = of_peak(of);
4755
4756 spin_lock(&memcg->peaks_lock);
4757
4758 usage = page_counter_read(pc);
4759 WRITE_ONCE(pc->local_watermark, usage);
4760
4761 list_for_each_entry(peer_ctx, watchers, list)
4762 if (usage > peer_ctx->value)
4763 WRITE_ONCE(peer_ctx->value, usage);
4764
4765 /* initial write, register watcher */
4766 if (ofp->value == OFP_PEAK_UNSET)
4767 list_add(&ofp->list, watchers);
4768
4769 WRITE_ONCE(ofp->value, usage);
4770 spin_unlock(&memcg->peaks_lock);
4771
4772 return nbytes;
4773}
4774
4775static ssize_t memory_peak_write(struct kernfs_open_file *of, char *buf,
4776 size_t nbytes, loff_t off)
4777{
4778 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4779
4780 return peak_write(of, buf, nbytes, off, &memcg->memory,
4781 &memcg->memory_peaks);
4782}
4783
4784#undef OFP_PEAK_UNSET
4785
4786static int memory_min_show(struct seq_file *m, void *v)
4787{
4788 return seq_puts_memcg_tunable(m,
4789 READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
4790}
4791
4792static ssize_t memory_min_write(struct kernfs_open_file *of,
4793 char *buf, size_t nbytes, loff_t off)
4794{
4795 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4796 unsigned long min;
4797 int err;
4798
4799 buf = strstrip(buf);
4800 err = page_counter_memparse(buf, "max", &min);
4801 if (err)
4802 return err;
4803
4804 page_counter_set_min(&memcg->memory, min);
4805
4806 return nbytes;
4807}
4808
4809static int memory_low_show(struct seq_file *m, void *v)
4810{
4811 return seq_puts_memcg_tunable(m,
4812 READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
4813}
4814
4815static ssize_t memory_low_write(struct kernfs_open_file *of,
4816 char *buf, size_t nbytes, loff_t off)
4817{
4818 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4819 unsigned long low;
4820 int err;
4821
4822 buf = strstrip(buf);
4823 err = page_counter_memparse(buf, "max", &low);
4824 if (err)
4825 return err;
4826
4827 page_counter_set_low(&memcg->memory, low);
4828
4829 return nbytes;
4830}
4831
4832static int memory_high_show(struct seq_file *m, void *v)
4833{
4834 return seq_puts_memcg_tunable(m,
4835 READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
4836}
4837
4838static ssize_t memory_high_write(struct kernfs_open_file *of,
4839 char *buf, size_t nbytes, loff_t off)
4840{
4841 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4842 unsigned int nr_retries = MAX_RECLAIM_RETRIES;
4843 bool drained = false;
4844 unsigned long high;
4845 int err;
4846
4847 buf = strstrip(buf);
4848 err = page_counter_memparse(buf, "max", &high);
4849 if (err)
4850 return err;
4851
4852 page_counter_set_high(&memcg->memory, high);
4853
4854 if (of->file->f_flags & O_NONBLOCK)
4855 goto out;
4856
4857 for (;;) {
4858 unsigned long nr_pages = page_counter_read(&memcg->memory);
4859 unsigned long reclaimed;
4860
4861 if (high != READ_ONCE(memcg->memory.high))
4862 break;
4863
4864 if (nr_pages <= high)
4865 break;
4866
4867 if (signal_pending(current))
4868 break;
4869
4870 /* cgroup_rmdir() waits for us with cgroup_mutex held. */
4871 if (memcg_is_dying(memcg))
4872 break;
4873
4874 if (!drained) {
4875 drain_all_stock(memcg);
4876 drained = true;
4877 continue;
4878 }
4879
4880 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
4881 GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL);
4882
4883 if (!reclaimed && !nr_retries--)
4884 break;
4885 }
4886out:
4887 memcg_wb_domain_size_changed(memcg);
4888 return nbytes;
4889}
4890
4891static int memory_max_show(struct seq_file *m, void *v)
4892{
4893 return seq_puts_memcg_tunable(m,
4894 READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
4895}
4896
4897static ssize_t memory_max_write(struct kernfs_open_file *of,
4898 char *buf, size_t nbytes, loff_t off)
4899{
4900 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4901 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
4902 bool drained = false;
4903 unsigned long max;
4904 int err;
4905
4906 buf = strstrip(buf);
4907 err = page_counter_memparse(buf, "max", &max);
4908 if (err)
4909 return err;
4910
4911 xchg(&memcg->memory.max, max);
4912
4913 if (of->file->f_flags & O_NONBLOCK)
4914 goto out;
4915
4916 for (;;) {
4917 unsigned long nr_pages = page_counter_read(&memcg->memory);
4918
4919 if (max != READ_ONCE(memcg->memory.max))
4920 break;
4921
4922 if (nr_pages <= max)
4923 break;
4924
4925 if (signal_pending(current))
4926 break;
4927
4928 /* cgroup_rmdir() waits for us with cgroup_mutex held. */
4929 if (memcg_is_dying(memcg))
4930 break;
4931
4932 if (!drained) {
4933 drain_all_stock(memcg);
4934 drained = true;
4935 continue;
4936 }
4937
4938 if (nr_reclaims) {
4939 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
4940 GFP_KERNEL, MEMCG_RECLAIM_MAY_SWAP, NULL))
4941 nr_reclaims--;
4942 continue;
4943 }
4944
4945 memcg_memory_event(memcg, MEMCG_OOM);
4946 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
4947 break;
4948 cond_resched();
4949 }
4950out:
4951 memcg_wb_domain_size_changed(memcg);
4952 return nbytes;
4953}
4954
4955/*
4956 * Note: don't forget to update the 'samples/cgroup/memcg_event_listener'
4957 * if any new events become available.
4958 */
4959static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
4960{
4961 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
4962 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
4963 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
4964 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
4965 seq_printf(m, "oom_kill %lu\n",
4966 atomic_long_read(&events[MEMCG_OOM_KILL]));
4967 seq_printf(m, "oom_group_kill %lu\n",
4968 atomic_long_read(&events[MEMCG_OOM_GROUP_KILL]));
4969 seq_printf(m, "sock_throttled %lu\n",
4970 atomic_long_read(&events[MEMCG_SOCK_THROTTLED]));
4971}
4972
4973static int memory_events_show(struct seq_file *m, void *v)
4974{
4975 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4976
4977 __memory_events_show(m, memcg->memory_events);
4978 return 0;
4979}
4980
4981static int memory_events_local_show(struct seq_file *m, void *v)
4982{
4983 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4984
4985 __memory_events_show(m, memcg->memory_events_local);
4986 return 0;
4987}
4988
4989int memory_stat_show(struct seq_file *m, void *v)
4990{
4991 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
4992 char *buf = kmalloc(SEQ_BUF_SIZE, GFP_KERNEL);
4993 struct seq_buf s;
4994
4995 if (!buf)
4996 return -ENOMEM;
4997 seq_buf_init(&s, buf, SEQ_BUF_SIZE);
4998 memory_stat_format(memcg, &s);
4999 seq_puts(m, buf);
5000 kfree(buf);
5001 return 0;
5002}
5003
5004#ifdef CONFIG_NUMA
5005static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
5006 int item)
5007{
5008 return lruvec_page_state(lruvec, item) *
5009 memcg_page_state_output_unit(item);
5010}
5011
5012static int memory_numa_stat_show(struct seq_file *m, void *v)
5013{
5014 int i;
5015 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
5016
5017 mem_cgroup_flush_stats(memcg);
5018
5019 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
5020 int nid;
5021
5022 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
5023 continue;
5024
5025 seq_printf(m, "%s", memory_stats[i].name);
5026 for_each_node_state(nid, N_MEMORY) {
5027 u64 size;
5028 struct lruvec *lruvec;
5029
5030 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
5031 size = lruvec_page_state_output(lruvec,
5032 memory_stats[i].idx);
5033 seq_printf(m, " N%d=%llu", nid, size);
5034 }
5035 seq_putc(m, '\n');
5036 }
5037
5038 return 0;
5039}
5040#endif
5041
5042static int memory_oom_group_show(struct seq_file *m, void *v)
5043{
5044 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
5045
5046 seq_printf(m, "%d\n", READ_ONCE(memcg->oom_group));
5047
5048 return 0;
5049}
5050
5051static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
5052 char *buf, size_t nbytes, loff_t off)
5053{
5054 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5055 int ret, oom_group;
5056
5057 buf = strstrip(buf);
5058 if (!buf)
5059 return -EINVAL;
5060
5061 ret = kstrtoint(buf, 0, &oom_group);
5062 if (ret)
5063 return ret;
5064
5065 if (oom_group != 0 && oom_group != 1)
5066 return -EINVAL;
5067
5068 WRITE_ONCE(memcg->oom_group, oom_group);
5069
5070 return nbytes;
5071}
5072
5073static ssize_t memory_reclaim(struct kernfs_open_file *of, char *buf,
5074 size_t nbytes, loff_t off)
5075{
5076 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5077 int ret;
5078
5079 ret = user_proactive_reclaim(buf, memcg, NULL);
5080 if (ret)
5081 return ret;
5082
5083 return nbytes;
5084}
5085
5086static struct cftype memory_files[] = {
5087 {
5088 .name = "current",
5089 .flags = CFTYPE_NOT_ON_ROOT,
5090 .read_u64 = memory_current_read,
5091 },
5092 {
5093 .name = "peak",
5094 .flags = CFTYPE_NOT_ON_ROOT,
5095 .open = peak_open,
5096 .release = peak_release,
5097 .seq_show = memory_peak_show,
5098 .write = memory_peak_write,
5099 },
5100 {
5101 .name = "min",
5102 .flags = CFTYPE_NOT_ON_ROOT,
5103 .seq_show = memory_min_show,
5104 .write = memory_min_write,
5105 },
5106 {
5107 .name = "low",
5108 .flags = CFTYPE_NOT_ON_ROOT,
5109 .seq_show = memory_low_show,
5110 .write = memory_low_write,
5111 },
5112 {
5113 .name = "high",
5114 .flags = CFTYPE_NOT_ON_ROOT,
5115 .seq_show = memory_high_show,
5116 .write = memory_high_write,
5117 },
5118 {
5119 .name = "max",
5120 .flags = CFTYPE_NOT_ON_ROOT,
5121 .seq_show = memory_max_show,
5122 .write = memory_max_write,
5123 },
5124 {
5125 .name = "events",
5126 .flags = CFTYPE_NOT_ON_ROOT,
5127 .file_offset = offsetof(struct mem_cgroup, events_file),
5128 .seq_show = memory_events_show,
5129 },
5130 {
5131 .name = "events.local",
5132 .flags = CFTYPE_NOT_ON_ROOT,
5133 .file_offset = offsetof(struct mem_cgroup, events_local_file),
5134 .seq_show = memory_events_local_show,
5135 },
5136 {
5137 .name = "stat",
5138 .seq_show = memory_stat_show,
5139 },
5140#ifdef CONFIG_NUMA
5141 {
5142 .name = "numa_stat",
5143 .seq_show = memory_numa_stat_show,
5144 },
5145#endif
5146 {
5147 .name = "oom.group",
5148 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
5149 .seq_show = memory_oom_group_show,
5150 .write = memory_oom_group_write,
5151 },
5152 {
5153 .name = "reclaim",
5154 .flags = CFTYPE_NS_DELEGATABLE,
5155 .write = memory_reclaim,
5156 },
5157 { } /* terminate */
5158};
5159
5160struct cgroup_subsys memory_cgrp_subsys = {
5161 .css_alloc = mem_cgroup_css_alloc,
5162 .css_online = mem_cgroup_css_online,
5163 .css_offline = mem_cgroup_css_offline,
5164 .css_released = mem_cgroup_css_released,
5165 .css_free = mem_cgroup_css_free,
5166 .css_reset = mem_cgroup_css_reset,
5167 .css_rstat_flush = mem_cgroup_css_rstat_flush,
5168 .attach = mem_cgroup_attach,
5169 .fork = mem_cgroup_fork,
5170 .exit = mem_cgroup_exit,
5171 .dfl_cftypes = memory_files,
5172#ifdef CONFIG_MEMCG_V1
5173 .legacy_cftypes = mem_cgroup_legacy_files,
5174#endif
5175 .early_init = 0,
5176};
5177
5178/**
5179 * mem_cgroup_calculate_protection - check if memory consumption is in the normal range
5180 * @root: the top ancestor of the sub-tree being checked
5181 * @memcg: the memory cgroup to check
5182 *
5183 * WARNING: This function is not stateless! It can only be used as part
5184 * of a top-down tree iteration, not for isolated queries.
5185 */
5186void mem_cgroup_calculate_protection(struct mem_cgroup *root,
5187 struct mem_cgroup *memcg)
5188{
5189 bool recursive_protection =
5190 cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT;
5191
5192 if (mem_cgroup_disabled())
5193 return;
5194
5195 if (!root)
5196 root = root_mem_cgroup;
5197
5198 page_counter_calculate_protection(&root->memory, &memcg->memory, recursive_protection);
5199}
5200
5201static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
5202 gfp_t gfp)
5203{
5204 int ret = 0;
5205 struct obj_cgroup *objcg;
5206
5207 objcg = get_obj_cgroup_from_memcg(memcg, folio_nid(folio));
5208 /* Do not account at the root objcg level. */
5209 if (!obj_cgroup_is_root(objcg))
5210 ret = try_charge_memcg(memcg, gfp, folio_nr_pages(folio));
5211 if (ret) {
5212 obj_cgroup_put(objcg);
5213 return ret;
5214 }
5215 commit_charge(folio, objcg);
5216 memcg1_commit_charge(folio, memcg);
5217
5218 return ret;
5219}
5220
5221int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp)
5222{
5223 struct mem_cgroup *memcg;
5224 int ret;
5225
5226 memcg = get_mem_cgroup_from_mm(mm);
5227 ret = charge_memcg(folio, memcg, gfp);
5228 css_put(&memcg->css);
5229
5230 return ret;
5231}
5232
5233/**
5234 * mem_cgroup_charge_hugetlb - charge the memcg for a hugetlb folio
5235 * @folio: folio being charged
5236 * @gfp: reclaim mode
5237 *
5238 * This function is called when allocating a huge page folio, after the page has
5239 * already been obtained and charged to the appropriate hugetlb cgroup
5240 * controller (if it is enabled).
5241 *
5242 * Returns ENOMEM if the memcg is already full.
5243 * Returns 0 if either the charge was successful, or if we skip the charging.
5244 */
5245int mem_cgroup_charge_hugetlb(struct folio *folio, gfp_t gfp)
5246{
5247 struct mem_cgroup *memcg = get_mem_cgroup_from_current();
5248 int ret = 0;
5249
5250 /*
5251 * Even memcg does not account for hugetlb, we still want to update
5252 * system-level stats via lruvec_stat_mod_folio. Return 0, and skip
5253 * charging the memcg.
5254 */
5255 if (mem_cgroup_disabled() || !memcg_accounts_hugetlb() ||
5256 !memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5257 goto out;
5258
5259 if (charge_memcg(folio, memcg, gfp))
5260 ret = -ENOMEM;
5261
5262out:
5263 mem_cgroup_put(memcg);
5264 return ret;
5265}
5266
5267/**
5268 * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin.
5269 * @folio: the folio to charge
5270 * @id: memory cgroup id
5271 * @mm: mm context of the victim
5272 * @gfp: reclaim mode
5273 *
5274 * This function charges a folio allocated for swapin. Please call this before
5275 * adding the folio to the swapcache.
5276 *
5277 * Returns 0 on success. Otherwise, an error code is returned.
5278 */
5279int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id,
5280 struct mm_struct *mm, gfp_t gfp)
5281{
5282 struct mem_cgroup *memcg;
5283 int ret;
5284
5285 if (mem_cgroup_disabled())
5286 return 0;
5287
5288 rcu_read_lock();
5289 memcg = mem_cgroup_from_private_id(id);
5290 if (!memcg || !css_tryget_online(&memcg->css))
5291 memcg = get_mem_cgroup_from_mm(mm);
5292 rcu_read_unlock();
5293
5294 ret = charge_memcg(folio, memcg, gfp);
5295
5296 css_put(&memcg->css);
5297 return ret;
5298}
5299
5300struct uncharge_gather {
5301 struct obj_cgroup *objcg;
5302 unsigned long nr_memory;
5303 unsigned long pgpgout;
5304 unsigned long nr_kmem;
5305 int nid;
5306};
5307
5308static inline void uncharge_gather_clear(struct uncharge_gather *ug)
5309{
5310 memset(ug, 0, sizeof(*ug));
5311}
5312
5313static void uncharge_batch(const struct uncharge_gather *ug)
5314{
5315 struct mem_cgroup *memcg;
5316
5317 rcu_read_lock();
5318 memcg = obj_cgroup_memcg(ug->objcg);
5319 if (ug->nr_memory) {
5320 memcg_uncharge(memcg, ug->nr_memory);
5321 if (ug->nr_kmem) {
5322 mod_memcg_state(memcg, MEMCG_KMEM, -ug->nr_kmem);
5323 memcg1_account_kmem(memcg, -ug->nr_kmem);
5324 }
5325 memcg1_oom_recover(memcg);
5326 }
5327
5328 memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory, ug->nid);
5329 rcu_read_unlock();
5330
5331 /* drop reference from uncharge_folio */
5332 obj_cgroup_put(ug->objcg);
5333}
5334
5335static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
5336{
5337 long nr_pages;
5338 struct obj_cgroup *objcg;
5339
5340 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
5341
5342 /*
5343 * Nobody should be changing or seriously looking at
5344 * folio objcg at this point, we have fully exclusive
5345 * access to the folio.
5346 */
5347 objcg = folio_objcg(folio);
5348 if (!objcg)
5349 return;
5350
5351 if (ug->objcg != objcg) {
5352 if (ug->objcg) {
5353 uncharge_batch(ug);
5354 uncharge_gather_clear(ug);
5355 }
5356 ug->objcg = objcg;
5357 ug->nid = folio_nid(folio);
5358
5359 /* pairs with obj_cgroup_put in uncharge_batch */
5360 obj_cgroup_get(objcg);
5361 }
5362
5363 nr_pages = folio_nr_pages(folio);
5364
5365 if (folio_memcg_kmem(folio)) {
5366 ug->nr_memory += nr_pages;
5367 ug->nr_kmem += nr_pages;
5368 } else {
5369 /* LRU pages aren't accounted at the root level */
5370 if (!obj_cgroup_is_root(objcg))
5371 ug->nr_memory += nr_pages;
5372 ug->pgpgout++;
5373
5374 WARN_ON_ONCE(folio_unqueue_deferred_split(folio));
5375 }
5376
5377 folio->memcg_data = 0;
5378 obj_cgroup_put(objcg);
5379}
5380
5381void __mem_cgroup_uncharge(struct folio *folio)
5382{
5383 struct uncharge_gather ug;
5384
5385 /* Don't touch folio->lru of any random page, pre-check: */
5386 if (!folio_memcg_charged(folio))
5387 return;
5388
5389 uncharge_gather_clear(&ug);
5390 uncharge_folio(folio, &ug);
5391 uncharge_batch(&ug);
5392}
5393
5394void __mem_cgroup_uncharge_folios(struct folio_batch *folios)
5395{
5396 struct uncharge_gather ug;
5397 unsigned int i;
5398
5399 uncharge_gather_clear(&ug);
5400 for (i = 0; i < folios->nr; i++)
5401 uncharge_folio(folios->folios[i], &ug);
5402 if (ug.objcg)
5403 uncharge_batch(&ug);
5404}
5405
5406/*
5407 * An LRU folio must hold the objcg belonging to its own node.
5408 *
5409 * memcg_reparent_objcgs() reparents a dying cgroup one node at a time: the
5410 * folios on that node's LRU lists move to the parent and that node's objcg is
5411 * redirected to the parent, atomically under the node's lru_lock.
5412 * folio_lruvec_lock() relies on this to provide a stable folio<->lruvec
5413 * binding. If a folio holds another node's objcg, its list membership and its
5414 * lruvec resolution change in separate lock sections, and an LRU operation in
5415 * between can re-add the folio to, and strand it on, the LRU list of a dead
5416 * memcg.
5417 *
5418 * So when migration transfers the memcg state to a folio on another node,
5419 * re-derive the objcg for the destination node. If the memcg is dying and the
5420 * destination node has already been reparented, the lookup walks up to the
5421 * nearest live ancestor - which is also where that node's LRU lists went.
5422 *
5423 * Returns the objcg to commit to @new, with a reference for the caller.
5424 */
5425static struct obj_cgroup *get_migration_objcg(struct folio *old,
5426 struct folio *new)
5427{
5428 struct obj_cgroup *old_objcg, *new_objcg;
5429 int new_nid = folio_nid(new);
5430
5431 old_objcg = get_obj_cgroup_from_folio(old);
5432
5433 if (folio_nid(old) == new_nid)
5434 return old_objcg;
5435
5436 rcu_read_lock();
5437 new_objcg = __get_obj_cgroup_from_memcg(obj_cgroup_memcg(old_objcg),
5438 new_nid);
5439 rcu_read_unlock();
5440
5441 obj_cgroup_put(old_objcg);
5442
5443 return new_objcg;
5444}
5445
5446/**
5447 * mem_cgroup_replace_folio - Charge a folio's replacement.
5448 * @old: Currently circulating folio.
5449 * @new: Replacement folio.
5450 *
5451 * Charge @new as a replacement folio for @old. @old will
5452 * be uncharged upon free.
5453 *
5454 * Both folios must be locked, @new->mapping must be set up.
5455 */
5456void mem_cgroup_replace_folio(struct folio *old, struct folio *new)
5457{
5458 struct mem_cgroup *memcg;
5459 struct obj_cgroup *objcg;
5460 long nr_pages = folio_nr_pages(new);
5461
5462 VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
5463 VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
5464 VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new);
5465 VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new);
5466
5467 if (mem_cgroup_disabled())
5468 return;
5469
5470 /* Page cache replacement: new folio already charged? */
5471 if (folio_memcg_charged(new))
5472 return;
5473
5474 VM_WARN_ON_ONCE_FOLIO(!folio_objcg(old), old);
5475 if (!folio_objcg(old))
5476 return;
5477
5478 objcg = get_migration_objcg(old, new);
5479
5480 rcu_read_lock();
5481 memcg = obj_cgroup_memcg(objcg);
5482
5483 /*
5484 * Force-charge the new page. The old one will be freed soon.
5485 *
5486 * The rootness of the committed objcg decides whether the final
5487 * uncharge of @new goes through the page counters (see
5488 * uncharge_folio()); charge them only if the uncharge will.
5489 */
5490 if (!obj_cgroup_is_root(objcg)) {
5491 page_counter_charge(&memcg->memory, nr_pages);
5492 if (do_memsw_account())
5493 page_counter_charge(&memcg->memsw, nr_pages);
5494 }
5495
5496 commit_charge(new, objcg);
5497 memcg1_commit_charge(new, memcg);
5498 rcu_read_unlock();
5499}
5500
5501/**
5502 * mem_cgroup_migrate - Transfer the memcg data from the old to the new folio.
5503 * @old: Currently circulating folio.
5504 * @new: Replacement folio.
5505 *
5506 * Transfer the memcg data from the old folio to the new folio for migration.
5507 * The old folio's data info will be cleared. The memory counters remain
5508 * unchanged, unless the charge moves out of a fully reparented ancestry
5509 * and has to be settled (see below).
5510 *
5511 * Both folios must be locked, @new->mapping must be set up.
5512 */
5513void mem_cgroup_migrate(struct folio *old, struct folio *new)
5514{
5515 struct obj_cgroup *objcg, *new_objcg;
5516
5517 VM_BUG_ON_FOLIO(!folio_test_locked(old), old);
5518 VM_BUG_ON_FOLIO(!folio_test_locked(new), new);
5519 VM_BUG_ON_FOLIO(folio_test_anon(old) != folio_test_anon(new), new);
5520 VM_BUG_ON_FOLIO(folio_nr_pages(old) != folio_nr_pages(new), new);
5521 VM_BUG_ON_FOLIO(folio_test_lru(old), old);
5522
5523 if (mem_cgroup_disabled())
5524 return;
5525
5526 objcg = folio_objcg(old);
5527 /*
5528 * Note that it is normal to see !objcg for a hugetlb folio.
5529 * For e.g, it could have been allocated when memory_hugetlb_accounting
5530 * was not selected.
5531 */
5532 VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(old) && !objcg, old);
5533 if (!objcg)
5534 return;
5535
5536 new_objcg = get_migration_objcg(old, new);
5537
5538 /*
5539 * @old was charged through a non-root objcg, so its charge is in the
5540 * page counters. If the re-derivation walked up to the root objcg -
5541 * @old's entire ancestry is dying and already reparented - the final
5542 * uncharge of @new will skip the page counters (see uncharge_folio()).
5543 * Settle them now: this is @old's eventual uncharge, moved up to the
5544 * point where its charge record ends.
5545 */
5546 if (obj_cgroup_is_root(new_objcg) && !obj_cgroup_is_root(objcg)) {
5547 rcu_read_lock();
5548 memcg_uncharge(obj_cgroup_memcg(objcg), folio_nr_pages(old));
5549 rcu_read_unlock();
5550 }
5551
5552 commit_charge(new, new_objcg);
5553
5554 /* Warning should never happen, so don't worry about refcount non-0 */
5555 WARN_ON_ONCE(folio_unqueue_deferred_split(old));
5556 old->memcg_data = 0;
5557
5558 /* @new holds its own reference now, drop @old's */
5559 obj_cgroup_put(objcg);
5560}
5561
5562DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5563EXPORT_SYMBOL(memcg_sockets_enabled_key);
5564
5565void mem_cgroup_sk_alloc(struct sock *sk)
5566{
5567 struct mem_cgroup *memcg;
5568
5569 if (!mem_cgroup_sockets_enabled)
5570 return;
5571
5572 /* Do not associate the sock with unrelated interrupted task's memcg. */
5573 if (!in_task())
5574 return;
5575
5576 rcu_read_lock();
5577 memcg = mem_cgroup_from_task(current);
5578 if (mem_cgroup_is_root(memcg))
5579 goto out;
5580 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg1_tcpmem_active(memcg))
5581 goto out;
5582 if (css_tryget(&memcg->css))
5583 sk->sk_memcg = memcg;
5584out:
5585 rcu_read_unlock();
5586}
5587
5588void mem_cgroup_sk_free(struct sock *sk)
5589{
5590 struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
5591
5592 if (memcg)
5593 css_put(&memcg->css);
5594}
5595
5596void mem_cgroup_sk_inherit(const struct sock *sk, struct sock *newsk)
5597{
5598 struct mem_cgroup *memcg;
5599
5600 if (sk->sk_memcg == newsk->sk_memcg)
5601 return;
5602
5603 mem_cgroup_sk_free(newsk);
5604
5605 memcg = mem_cgroup_from_sk(sk);
5606 if (memcg)
5607 css_get(&memcg->css);
5608
5609 newsk->sk_memcg = sk->sk_memcg;
5610}
5611
5612/**
5613 * mem_cgroup_sk_charge - charge socket memory
5614 * @sk: socket in memcg to charge
5615 * @nr_pages: number of pages to charge
5616 * @gfp_mask: reclaim mode
5617 *
5618 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
5619 * @memcg's configured limit, %false if it doesn't.
5620 */
5621bool mem_cgroup_sk_charge(const struct sock *sk, unsigned int nr_pages,
5622 gfp_t gfp_mask)
5623{
5624 struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
5625
5626 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
5627 return memcg1_charge_skmem(memcg, nr_pages, gfp_mask);
5628
5629 if (try_charge_memcg(memcg, gfp_mask, nr_pages) == 0) {
5630 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
5631 return true;
5632 }
5633
5634 return false;
5635}
5636
5637/**
5638 * mem_cgroup_sk_uncharge - uncharge socket memory
5639 * @sk: socket in memcg to uncharge
5640 * @nr_pages: number of pages to uncharge
5641 */
5642void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages)
5643{
5644 struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
5645
5646 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5647 memcg1_uncharge_skmem(memcg, nr_pages);
5648 return;
5649 }
5650
5651 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
5652
5653 refill_stock(memcg, nr_pages);
5654}
5655
5656void mem_cgroup_flush_workqueue(void)
5657{
5658 flush_workqueue(memcg_wq);
5659}
5660
5661static int __init cgroup_memory(char *s)
5662{
5663 char *token;
5664
5665 while ((token = strsep(&s, ",")) != NULL) {
5666 if (!*token)
5667 continue;
5668 if (!strcmp(token, "nosocket"))
5669 cgroup_memory_nosocket = true;
5670 if (!strcmp(token, "nokmem"))
5671 cgroup_memory_nokmem = true;
5672 if (!strcmp(token, "nobpf"))
5673 cgroup_memory_nobpf = true;
5674 }
5675 return 1;
5676}
5677__setup("cgroup.memory=", cgroup_memory);
5678
5679/*
5680 * Memory controller init before cgroup_init() initialize root_mem_cgroup.
5681 *
5682 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
5683 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
5684 * basically everything that doesn't depend on a specific mem_cgroup structure
5685 * should be initialized from here.
5686 */
5687int __init mem_cgroup_init(void)
5688{
5689 unsigned int memcg_size;
5690 int cpu;
5691
5692 /*
5693 * Currently s32 type (can refer to struct batched_lruvec_stat) is
5694 * used for per-memcg-per-cpu caching of per-node statistics. In order
5695 * to work fine, we should make sure that the overfill threshold can't
5696 * exceed S32_MAX / PAGE_SIZE.
5697 */
5698 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
5699
5700 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
5701 memcg_hotplug_cpu_dead);
5702
5703 memcg_wq = alloc_workqueue("memcg", WQ_PERCPU, 0);
5704 WARN_ON(!memcg_wq);
5705
5706 for_each_possible_cpu(cpu) {
5707 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5708 drain_local_memcg_stock);
5709 INIT_WORK(&per_cpu_ptr(&obj_stock, cpu)->work,
5710 drain_local_obj_stock);
5711 }
5712
5713 memcg_size = struct_size_t(struct mem_cgroup, nodeinfo, nr_node_ids);
5714 memcg_cachep = kmem_cache_create("mem_cgroup", memcg_size, 0,
5715 SLAB_PANIC | SLAB_HWCACHE_ALIGN, NULL);
5716
5717 memcg_pn_cachep = KMEM_CACHE(mem_cgroup_per_node,
5718 SLAB_PANIC | SLAB_HWCACHE_ALIGN);
5719
5720 return 0;
5721}
5722
5723#ifdef CONFIG_SWAP
5724/**
5725 * __mem_cgroup_try_charge_swap - try charging swap space for a folio
5726 * @folio: folio being added to swap
5727 *
5728 * Try to charge @folio's memcg for the swap space at folio->swap.
5729 *
5730 * Returns 0 on success, -ENOMEM on failure.
5731 */
5732int __mem_cgroup_try_charge_swap(struct folio *folio)
5733{
5734 unsigned int nr_pages = folio_nr_pages(folio);
5735 struct swap_cluster_info *ci;
5736 struct page_counter *counter;
5737 struct mem_cgroup *memcg;
5738 struct obj_cgroup *objcg;
5739
5740 if (do_memsw_account())
5741 return 0;
5742
5743 objcg = folio_objcg(folio);
5744 VM_WARN_ON_ONCE_FOLIO(!objcg, folio);
5745 if (!objcg)
5746 return 0;
5747
5748 rcu_read_lock();
5749 memcg = obj_cgroup_memcg(objcg);
5750 if (!folio_test_swapcache(folio)) {
5751 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
5752 rcu_read_unlock();
5753 return 0;
5754 }
5755
5756 memcg = mem_cgroup_private_id_get_online(memcg, nr_pages);
5757 /* memcg is pined by memcg ID. */
5758 rcu_read_unlock();
5759
5760 if (!mem_cgroup_is_root(memcg) &&
5761 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
5762 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
5763 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
5764 mem_cgroup_private_id_put(memcg, nr_pages);
5765 return -ENOMEM;
5766 }
5767 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
5768
5769 ci = swap_cluster_get_and_lock(folio);
5770 __swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_pages,
5771 mem_cgroup_private_id(memcg));
5772 swap_cluster_unlock(ci);
5773
5774 return 0;
5775}
5776
5777/**
5778 * __mem_cgroup_uncharge_swap - uncharge swap space
5779 * @id: cgroup id to uncharge
5780 * @nr_pages: the amount of swap space to uncharge
5781 */
5782void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages)
5783{
5784 struct mem_cgroup *memcg;
5785
5786 rcu_read_lock();
5787 memcg = mem_cgroup_from_private_id(id);
5788 if (memcg) {
5789 if (!mem_cgroup_is_root(memcg)) {
5790 if (do_memsw_account())
5791 page_counter_uncharge(&memcg->memsw, nr_pages);
5792 else
5793 page_counter_uncharge(&memcg->swap, nr_pages);
5794 }
5795 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
5796 mem_cgroup_private_id_put(memcg, nr_pages);
5797 }
5798 rcu_read_unlock();
5799}
5800
5801long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
5802{
5803 long nr_swap_pages = get_nr_swap_pages();
5804
5805 if (mem_cgroup_disabled() || do_memsw_account())
5806 return nr_swap_pages;
5807 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg))
5808 nr_swap_pages = min_t(long, nr_swap_pages,
5809 READ_ONCE(memcg->swap.max) -
5810 page_counter_read(&memcg->swap));
5811 return nr_swap_pages;
5812}
5813
5814bool mem_cgroup_swap_full(struct folio *folio)
5815{
5816 struct mem_cgroup *memcg;
5817 bool ret = false;
5818
5819 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
5820
5821 if (vm_swap_full())
5822 return true;
5823 if (do_memsw_account() || !folio_memcg_charged(folio))
5824 return ret;
5825
5826 rcu_read_lock();
5827 memcg = folio_memcg(folio);
5828 for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
5829 unsigned long usage = page_counter_read(&memcg->swap);
5830
5831 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
5832 usage * 2 >= READ_ONCE(memcg->swap.max)) {
5833 ret = true;
5834 break;
5835 }
5836 }
5837 rcu_read_unlock();
5838
5839 return ret;
5840}
5841
5842static int __init setup_swap_account(char *s)
5843{
5844 bool res;
5845
5846 if (!kstrtobool(s, &res) && !res)
5847 pr_warn_once("The swapaccount=0 commandline option is deprecated "
5848 "in favor of configuring swap control via cgroupfs. "
5849 "Please report your usecase to [email protected] if you "
5850 "depend on this functionality.\n");
5851 return 1;
5852}
5853__setup("swapaccount=", setup_swap_account);
5854
5855static u64 swap_current_read(struct cgroup_subsys_state *css,
5856 struct cftype *cft)
5857{
5858 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5859
5860 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
5861}
5862
5863static int swap_peak_show(struct seq_file *sf, void *v)
5864{
5865 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
5866
5867 return peak_show(sf, v, &memcg->swap);
5868}
5869
5870static ssize_t swap_peak_write(struct kernfs_open_file *of, char *buf,
5871 size_t nbytes, loff_t off)
5872{
5873 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5874
5875 return peak_write(of, buf, nbytes, off, &memcg->swap,
5876 &memcg->swap_peaks);
5877}
5878
5879static int swap_high_show(struct seq_file *m, void *v)
5880{
5881 return seq_puts_memcg_tunable(m,
5882 READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
5883}
5884
5885static ssize_t swap_high_write(struct kernfs_open_file *of,
5886 char *buf, size_t nbytes, loff_t off)
5887{
5888 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5889 unsigned long high;
5890 int err;
5891
5892 buf = strstrip(buf);
5893 err = page_counter_memparse(buf, "max", &high);
5894 if (err)
5895 return err;
5896
5897 page_counter_set_high(&memcg->swap, high);
5898
5899 return nbytes;
5900}
5901
5902static int swap_max_show(struct seq_file *m, void *v)
5903{
5904 return seq_puts_memcg_tunable(m,
5905 READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
5906}
5907
5908static ssize_t swap_max_write(struct kernfs_open_file *of,
5909 char *buf, size_t nbytes, loff_t off)
5910{
5911 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5912 unsigned long max;
5913 int err;
5914
5915 buf = strstrip(buf);
5916 err = page_counter_memparse(buf, "max", &max);
5917 if (err)
5918 return err;
5919
5920 xchg(&memcg->swap.max, max);
5921
5922 return nbytes;
5923}
5924
5925static int swap_events_show(struct seq_file *m, void *v)
5926{
5927 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
5928
5929 seq_printf(m, "high %lu\n",
5930 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
5931 seq_printf(m, "max %lu\n",
5932 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
5933 seq_printf(m, "fail %lu\n",
5934 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
5935
5936 return 0;
5937}
5938
5939static struct cftype swap_files[] = {
5940 {
5941 .name = "swap.current",
5942 .flags = CFTYPE_NOT_ON_ROOT,
5943 .read_u64 = swap_current_read,
5944 },
5945 {
5946 .name = "swap.high",
5947 .flags = CFTYPE_NOT_ON_ROOT,
5948 .seq_show = swap_high_show,
5949 .write = swap_high_write,
5950 },
5951 {
5952 .name = "swap.max",
5953 .flags = CFTYPE_NOT_ON_ROOT,
5954 .seq_show = swap_max_show,
5955 .write = swap_max_write,
5956 },
5957 {
5958 .name = "swap.peak",
5959 .flags = CFTYPE_NOT_ON_ROOT,
5960 .open = peak_open,
5961 .release = peak_release,
5962 .seq_show = swap_peak_show,
5963 .write = swap_peak_write,
5964 },
5965 {
5966 .name = "swap.events",
5967 .flags = CFTYPE_NOT_ON_ROOT,
5968 .file_offset = offsetof(struct mem_cgroup, swap_events_file),
5969 .seq_show = swap_events_show,
5970 },
5971 { } /* terminate */
5972};
5973
5974#ifdef CONFIG_ZSWAP
5975/**
5976 * obj_cgroup_may_zswap - check if this cgroup can zswap
5977 * @objcg: the object cgroup
5978 *
5979 * Check if the hierarchical zswap limit has been reached.
5980 *
5981 * This doesn't check for specific headroom, and it is not atomic
5982 * either. But with zswap, the size of the allocation is only known
5983 * once compression has occurred, and this optimistic pre-check avoids
5984 * spending cycles on compression when there is already no room left
5985 * or zswap is disabled altogether somewhere in the hierarchy.
5986 */
5987bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
5988{
5989 struct mem_cgroup *memcg, *original_memcg;
5990 bool ret = true;
5991
5992 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
5993 return true;
5994
5995 original_memcg = get_mem_cgroup_from_objcg(objcg);
5996 for (memcg = original_memcg; !mem_cgroup_is_root(memcg);
5997 memcg = parent_mem_cgroup(memcg)) {
5998 unsigned long max = READ_ONCE(memcg->zswap_max);
5999 unsigned long pages;
6000
6001 if (max == PAGE_COUNTER_MAX)
6002 continue;
6003 if (max == 0) {
6004 ret = false;
6005 break;
6006 }
6007
6008 /* Force flush to get accurate stats for charging */
6009 __mem_cgroup_flush_stats(memcg, true);
6010 pages = memcg_page_state(memcg, MEMCG_ZSWAP_B) / PAGE_SIZE;
6011 if (pages < max)
6012 continue;
6013 ret = false;
6014 break;
6015 }
6016 mem_cgroup_put(original_memcg);
6017 return ret;
6018}
6019
6020/**
6021 * obj_cgroup_charge_zswap - charge compression backend memory
6022 * @objcg: the object cgroup
6023 * @size: size of compressed object
6024 *
6025 * This forces the charge after obj_cgroup_may_zswap() allowed
6026 * compression and storage in zswap for this cgroup to go ahead.
6027 */
6028void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
6029{
6030 struct mem_cgroup *memcg;
6031
6032 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
6033 return;
6034
6035 if (obj_cgroup_is_root(objcg))
6036 return;
6037
6038 VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC));
6039
6040 /* PF_MEMALLOC context, charging must succeed */
6041 if (obj_cgroup_charge(objcg, GFP_KERNEL, size))
6042 VM_WARN_ON_ONCE(1);
6043
6044 rcu_read_lock();
6045 memcg = obj_cgroup_memcg(objcg);
6046 mod_memcg_state(memcg, MEMCG_ZSWAP_B, size);
6047 mod_memcg_state(memcg, MEMCG_ZSWAPPED, 1);
6048 if (size == PAGE_SIZE)
6049 mod_memcg_state(memcg, MEMCG_ZSWAP_INCOMP, 1);
6050 rcu_read_unlock();
6051}
6052
6053/**
6054 * obj_cgroup_uncharge_zswap - uncharge compression backend memory
6055 * @objcg: the object cgroup
6056 * @size: size of compressed object
6057 *
6058 * Uncharges zswap memory on page in.
6059 */
6060void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)
6061{
6062 struct mem_cgroup *memcg;
6063
6064 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
6065 return;
6066
6067 if (obj_cgroup_is_root(objcg))
6068 return;
6069
6070 obj_cgroup_uncharge(objcg, size);
6071
6072 rcu_read_lock();
6073 memcg = obj_cgroup_memcg(objcg);
6074 mod_memcg_state(memcg, MEMCG_ZSWAP_B, -size);
6075 mod_memcg_state(memcg, MEMCG_ZSWAPPED, -1);
6076 if (size == PAGE_SIZE)
6077 mod_memcg_state(memcg, MEMCG_ZSWAP_INCOMP, -1);
6078 rcu_read_unlock();
6079}
6080
6081bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)
6082{
6083 /* if zswap is disabled, do not block pages going to the swapping device */
6084 if (!zswap_is_enabled())
6085 return true;
6086
6087 for (; memcg; memcg = parent_mem_cgroup(memcg))
6088 if (!READ_ONCE(memcg->zswap_writeback))
6089 return false;
6090
6091 return true;
6092}
6093
6094static u64 zswap_current_read(struct cgroup_subsys_state *css,
6095 struct cftype *cft)
6096{
6097 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6098
6099 mem_cgroup_flush_stats(memcg);
6100 return memcg_page_state(memcg, MEMCG_ZSWAP_B);
6101}
6102
6103static int zswap_max_show(struct seq_file *m, void *v)
6104{
6105 return seq_puts_memcg_tunable(m,
6106 READ_ONCE(mem_cgroup_from_seq(m)->zswap_max));
6107}
6108
6109static ssize_t zswap_max_write(struct kernfs_open_file *of,
6110 char *buf, size_t nbytes, loff_t off)
6111{
6112 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6113 unsigned long max;
6114 int err;
6115
6116 buf = strstrip(buf);
6117 err = page_counter_memparse(buf, "max", &max);
6118 if (err)
6119 return err;
6120
6121 xchg(&memcg->zswap_max, max);
6122
6123 return nbytes;
6124}
6125
6126static int zswap_writeback_show(struct seq_file *m, void *v)
6127{
6128 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6129
6130 seq_printf(m, "%d\n", READ_ONCE(memcg->zswap_writeback));
6131 return 0;
6132}
6133
6134static ssize_t zswap_writeback_write(struct kernfs_open_file *of,
6135 char *buf, size_t nbytes, loff_t off)
6136{
6137 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6138 int zswap_writeback;
6139 ssize_t parse_ret = kstrtoint(strstrip(buf), 0, &zswap_writeback);
6140
6141 if (parse_ret)
6142 return parse_ret;
6143
6144 if (zswap_writeback != 0 && zswap_writeback != 1)
6145 return -EINVAL;
6146
6147 WRITE_ONCE(memcg->zswap_writeback, zswap_writeback);
6148 return nbytes;
6149}
6150
6151static struct cftype zswap_files[] = {
6152 {
6153 .name = "zswap.current",
6154 .flags = CFTYPE_NOT_ON_ROOT,
6155 .read_u64 = zswap_current_read,
6156 },
6157 {
6158 .name = "zswap.max",
6159 .flags = CFTYPE_NOT_ON_ROOT,
6160 .seq_show = zswap_max_show,
6161 .write = zswap_max_write,
6162 },
6163 {
6164 .name = "zswap.writeback",
6165 .seq_show = zswap_writeback_show,
6166 .write = zswap_writeback_write,
6167 },
6168 { } /* terminate */
6169};
6170#endif /* CONFIG_ZSWAP */
6171
6172static int __init mem_cgroup_swap_init(void)
6173{
6174 if (mem_cgroup_disabled())
6175 return 0;
6176
6177 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
6178#ifdef CONFIG_MEMCG_V1
6179 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
6180#endif
6181#ifdef CONFIG_ZSWAP
6182 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, zswap_files));
6183#endif
6184 return 0;
6185}
6186subsys_initcall(mem_cgroup_swap_init);
6187
6188#endif /* CONFIG_SWAP */
6189
6190void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask)
6191{
6192 nodemask_t allowed;
6193
6194 if (!memcg)
6195 return;
6196
6197 /*
6198 * Since this interface is intended for use by migration paths, and
6199 * reclaim and migration are subject to race conditions such as changes
6200 * in effective_mems and hot-unpluging of nodes, inaccurate allowed
6201 * mask is acceptable.
6202 */
6203 cpuset_nodes_allowed(memcg->css.cgroup, &allowed);
6204 nodes_and(*mask, *mask, allowed);
6205}
6206
6207void mem_cgroup_show_protected_memory(struct mem_cgroup *memcg)
6208{
6209 if (mem_cgroup_disabled() || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6210 return;
6211
6212 if (!memcg)
6213 memcg = root_mem_cgroup;
6214
6215 pr_warn("Memory cgroup min protection %lukB -- low protection %lukB",
6216 K(atomic_long_read(&memcg->memory.children_min_usage)),
6217 K(atomic_long_read(&memcg->memory.children_low_usage)));
6218}