projects/linux·blob master · refs

10137 lines · 251.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * SLUB: A slab allocator with low overhead percpu array caches and mostly
4 * lockless freeing of objects to slabs in the slowpath.
5 *
6 * The allocator synchronizes using spin_trylock for percpu arrays in the
7 * fastpath, and cmpxchg_double (or bit spinlock) for slowpath freeing.
8 * Uses a centralized lock to manage a pool of partial slabs.
9 *
10 * (C) 2007 SGI, Christoph Lameter
11 * (C) 2011 Linux Foundation, Christoph Lameter
12 * (C) 2025 SUSE, Vlastimil Babka
13 */
14
15#include <linux/mm.h>
16#include <linux/swap.h> /* mm_account_reclaimed_pages() */
17#include <linux/module.h>
18#include <linux/bit_spinlock.h>
19#include <linux/interrupt.h>
20#include <linux/swab.h>
21#include <linux/bitops.h>
22#include <linux/slab.h>
23#include "slab.h"
24#include <linux/vmalloc.h>
25#include <linux/proc_fs.h>
26#include <linux/seq_file.h>
27#include <linux/kasan.h>
28#include <linux/node.h>
29#include <linux/kmsan.h>
30#include <linux/cpu.h>
31#include <linux/cpuset.h>
32#include <linux/mempolicy.h>
33#include <linux/ctype.h>
34#include <linux/stackdepot.h>
35#include <linux/debugobjects.h>
36#include <linux/kallsyms.h>
37#include <linux/kfence.h>
38#include <linux/memory.h>
39#include <linux/math64.h>
40#include <linux/fault-inject.h>
41#include <linux/kmemleak.h>
42#include <linux/stacktrace.h>
43#include <linux/prefetch.h>
44#include <linux/memcontrol.h>
45#include <linux/random.h>
46#include <linux/prandom.h>
47#include <kunit/test.h>
48#include <kunit/test-bug.h>
49#include <linux/sort.h>
50#include <linux/irq_work.h>
51#include <linux/kprobes.h>
52#include <linux/debugfs.h>
53#include <trace/events/kmem.h>
54
55#include "internal.h"
56#include "page_alloc.h"
57
58/*
59 * Lock order:
60 * 0. cpu_hotplug_lock
61 * 1. slab_mutex (Global Mutex)
62 * 2a. kmem_cache->cpu_sheaves->lock (Local trylock)
63 * 2b. barn->lock (Spinlock)
64 * 2c. node->list_lock (Spinlock)
65 * 3. slab_lock(slab) (Only on some arches)
66 * 4. object_map_lock (Only for debugging)
67 *
68 * slab_mutex
69 *
70 * The role of the slab_mutex is to protect the list of all the slabs
71 * and to synchronize major metadata changes to slab cache structures.
72 * Also synchronizes memory hotplug callbacks.
73 *
74 * slab_lock
75 *
76 * The slab_lock is a wrapper around the page lock, thus it is a bit
77 * spinlock.
78 *
79 * The slab_lock is only used on arches that do not have the ability
80 * to do a cmpxchg_double. It only protects:
81 *
82 * A. slab->freelist -> List of free objects in a slab
83 * B. slab->inuse -> Number of objects in use
84 * C. slab->objects -> Number of objects in slab
85 * D. slab->frozen -> frozen state
86 *
87 * SL_partial slabs
88 *
89 * Slabs on node partial list have at least one free object. A limited number
90 * of slabs on the list can be fully free (slab->inuse == 0), until we start
91 * discarding them. These slabs are marked with SL_partial, and the flag is
92 * cleared while removing them, usually to grab their freelist afterwards.
93 * This clearing also exempts them from list management. Please see
94 * __slab_free() for more details.
95 *
96 * Full slabs
97 *
98 * For caches without debugging enabled, full slabs (slab->inuse ==
99 * slab->objects and slab->freelist == NULL) are not placed on any list.
100 * The __slab_free() freeing the first object from such a slab will place
101 * it on the partial list. Caches with debugging enabled place such slab
102 * on the full list and use different allocation and freeing paths.
103 *
104 * Frozen slabs
105 *
106 * If a slab is frozen then it is exempt from list management. It is used to
107 * indicate a slab that has failed consistency checks and thus cannot be
108 * allocated from anymore - it is also marked as full. Any previously
109 * allocated objects will be simply leaked upon freeing instead of attempting
110 * to modify the potentially corrupted freelist and metadata.
111 *
112 * To sum up, the current scheme is:
113 * - node partial slab: SL_partial && !full && !frozen
114 * - taken off partial list: !SL_partial && !full && !frozen
115 * - full slab, not on any list: !SL_partial && full && !frozen
116 * - frozen due to inconsistency: !SL_partial && full && frozen
117 *
118 * node->list_lock (spinlock)
119 *
120 * The list_lock protects the partial and full list on each node and
121 * the partial slab counter. If taken then no new slabs may be added or
122 * removed from the lists nor make the number of partial slabs be modified.
123 * (Note that the total number of slabs is an atomic value that may be
124 * modified without taking the list lock).
125 *
126 * The list_lock is a centralized lock and thus we avoid taking it as
127 * much as possible. As long as SLUB does not have to handle partial
128 * slabs, operations can continue without any centralized lock.
129 *
130 * For debug caches, all allocations are forced to go through a list_lock
131 * protected region to serialize against concurrent validation.
132 *
133 * cpu_sheaves->lock (local_trylock)
134 *
135 * This lock protects fastpath operations on the percpu sheaves. On !RT it
136 * only disables preemption and does no atomic operations. As long as the main
137 * or spare sheaf can handle the allocation or free, there is no other
138 * overhead.
139 *
140 * barn->lock (spinlock)
141 *
142 * This lock protects the operations on per-NUMA-node barn. It can quickly
143 * serve an empty or full sheaf if available, and avoid more expensive refill
144 * or flush operation.
145 *
146 * Lockless freeing
147 *
148 * Objects may have to be freed to their slabs when they are from a remote
149 * node (where we want to avoid filling local sheaves with remote objects)
150 * or when there are too many full sheaves. On architectures supporting
151 * cmpxchg_double this is done by a lockless update of slab's freelist and
152 * counters, otherwise slab_lock is taken. This only needs to take the
153 * list_lock if it's a first free to a full slab, or when a slab becomes empty
154 * after the free.
155 *
156 * irq, preemption, migration considerations
157 *
158 * Interrupts are disabled as part of list_lock or barn lock operations, or
159 * around the slab_lock operation, in order to make the slab allocator safe
160 * to use in the context of an irq.
161 * Preemption is disabled as part of local_trylock operations.
162 * kmalloc_nolock() and kfree_nolock() are safe in NMI context but see
163 * their limitations.
164 *
165 * SLUB assigns two object arrays called sheaves for caching allocations and
166 * frees on each cpu, with a NUMA node shared barn for balancing between cpus.
167 * Allocations and frees are primarily served from these sheaves.
168 *
169 * Slabs with free elements are kept on a partial list and during regular
170 * operations no list for full slabs is used. If an object in a full slab is
171 * freed then the slab will show up again on the partial lists.
172 * We track full slabs for debugging purposes though because otherwise we
173 * cannot scan all objects.
174 *
175 * Slabs are freed when they become empty. Teardown and setup is minimal so we
176 * rely on the page allocators per cpu caches for fast frees and allocs.
177 *
178 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
179 * options set. This moves slab handling out of
180 * the fast path and disables lockless freelists.
181 */
182
183/**
184 * enum slab_flags - How the slab flags bits are used.
185 * @SL_locked: Is locked with slab_lock()
186 * @SL_partial: On the per-node partial list
187 * @SL_pfmemalloc: Was allocated from PF_MEMALLOC reserves
188 *
189 * The slab flags share space with the page flags but some bits have
190 * different interpretations. The high bits are used for information
191 * like zone/node/section.
192 */
193enum slab_flags {
194 SL_locked = PG_locked,
195 SL_partial = PG_workingset, /* Historical reasons for this bit */
196 SL_pfmemalloc = PG_active, /* Historical reasons for this bit */
197};
198
199#ifndef CONFIG_SLUB_TINY
200#define __fastpath_inline __always_inline
201#else
202#define __fastpath_inline
203#endif
204
205#ifdef CONFIG_SLUB_DEBUG
206#ifdef CONFIG_SLUB_DEBUG_ON
207DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
208#else
209DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
210#endif
211#endif /* CONFIG_SLUB_DEBUG */
212
213#ifdef CONFIG_NUMA
214static DEFINE_STATIC_KEY_FALSE(strict_numa);
215#endif
216
217#ifdef CONFIG_MEM_ALLOC_PROFILING
218DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
219 slab_obj_ext_has_codetag_key);
220#endif
221
222/* Structure holding extra parameters for slab allocations */
223struct slab_alloc_context {
224 unsigned long caller_addr;
225 size_t orig_size;
226 unsigned int alloc_flags;
227 struct list_lru *lru;
228};
229
230/* Structure holding parameters for get_partial_node_bulk() */
231struct partial_bulk_context {
232 gfp_t flags;
233 unsigned int min_objects;
234 unsigned int max_objects;
235 struct list_head slabs;
236};
237
238/* Structure used to iterate over objects within a slab */
239struct slab_obj_iter {
240 unsigned long pos;
241 void *start;
242#ifdef CONFIG_SLAB_FREELIST_RANDOM
243 unsigned long freelist_count;
244 unsigned long page_limit;
245 bool random;
246#endif
247};
248
249static inline bool kmem_cache_debug(struct kmem_cache *s)
250{
251 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
252}
253
254void *fixup_red_left(struct kmem_cache *s, void *p)
255{
256 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
257 p += s->red_left_pad;
258
259 return p;
260}
261
262/*
263 * Issues still to be resolved:
264 *
265 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
266 *
267 * - Variable sizing of the per node arrays
268 */
269
270/* Enable to log cmpxchg failures */
271#undef SLUB_DEBUG_CMPXCHG
272
273#ifndef CONFIG_SLUB_TINY
274/*
275 * Minimum number of partial slabs. These will be left on the partial
276 * lists even if they are empty. kmem_cache_shrink may reclaim them.
277 */
278#define MIN_PARTIAL 5
279
280/*
281 * Maximum number of desirable partial slabs.
282 * The existence of more partial slabs makes kmem_cache_shrink
283 * sort the partial list by the number of objects in use.
284 */
285#define MAX_PARTIAL 10
286#else
287#define MIN_PARTIAL 0
288#define MAX_PARTIAL 0
289#endif
290
291#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
292 SLAB_POISON | SLAB_STORE_USER)
293
294/*
295 * These debug flags cannot use CMPXCHG because there might be consistency
296 * issues when checking or reading debug information
297 */
298#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
299 SLAB_TRACE)
300
301
302/*
303 * Debugging flags that require metadata to be stored in the slab. These get
304 * disabled when slab_debug=O is used and a cache's min order increases with
305 * metadata.
306 */
307#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
308
309#define OO_SHIFT 16
310#define OO_MASK ((1 << OO_SHIFT) - 1)
311#define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */
312
313/* Internal SLUB flags */
314/* Poison object */
315#define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON)
316/* Use cmpxchg_double */
317
318#ifdef system_has_freelist_aba
319#define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE)
320#else
321#define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED
322#endif
323
324/*
325 * Tracking user of a slab.
326 */
327#define TRACK_ADDRS_COUNT 16
328struct track {
329 unsigned long addr; /* Called from address */
330#ifdef CONFIG_STACKDEPOT
331 depot_stack_handle_t handle;
332#endif
333 int cpu; /* Was running on cpu */
334 int pid; /* Pid context */
335 unsigned long when; /* When did the operation occur */
336};
337
338enum track_item { TRACK_ALLOC, TRACK_FREE };
339
340#ifdef SLAB_SUPPORTS_SYSFS
341static int sysfs_slab_add(struct kmem_cache *);
342static int __init slab_kset_init(void);
343static void __init slab_sysfs_process_aliases(void);
344#else
345static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
346static inline int slab_kset_init(void) { return 0; }
347static inline void slab_sysfs_process_aliases(void) { }
348#endif
349
350#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
351static void debugfs_slab_add(struct kmem_cache *);
352static void __init slab_debugfs_root_init(void);
353#else
354static inline void debugfs_slab_add(struct kmem_cache *s) { }
355static inline void slab_debugfs_root_init(void) { }
356#endif
357
358enum add_mode {
359 ADD_TO_HEAD,
360 ADD_TO_TAIL,
361};
362
363enum stat_item {
364 ALLOC_FASTPATH, /* Allocation from percpu sheaves */
365 ALLOC_SLOWPATH, /* Allocation from partial or new slab */
366 FREE_RCU_SHEAF, /* Free to rcu_free sheaf */
367 FREE_RCU_SHEAF_FAIL, /* Failed to free to a rcu_free sheaf */
368 FREE_FASTPATH, /* Free to percpu sheaves */
369 FREE_SLOWPATH, /* Free to a slab */
370 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */
371 FREE_REMOVE_PARTIAL, /* Freeing removes last object */
372 ALLOC_SLAB, /* New slab acquired from page allocator */
373 ALLOC_NODE_MISMATCH, /* Requested node different from cpu sheaf */
374 FREE_SLAB, /* Slab freed to the page allocator */
375 ORDER_FALLBACK, /* Number of times fallback was necessary */
376 CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */
377 SHEAF_FLUSH, /* Objects flushed from a sheaf */
378 SHEAF_REFILL, /* Objects refilled to a sheaf */
379 SHEAF_ALLOC, /* Allocation of an empty sheaf including oversized ones */
380 SHEAF_FREE, /* Freeing of an empty sheaf including oversized ones */
381 BARN_GET, /* Got full sheaf from barn */
382 BARN_GET_FAIL, /* Failed to get full sheaf from barn */
383 BARN_PUT, /* Put full sheaf to barn */
384 BARN_PUT_FAIL, /* Failed to put full sheaf to barn */
385 SHEAF_PREFILL_FAST, /* Sheaf prefill grabbed the spare sheaf */
386 SHEAF_PREFILL_SLOW, /* Sheaf prefill found no spare sheaf */
387 SHEAF_PREFILL_OVERSIZE, /* Allocation of oversize sheaf for prefill */
388 SHEAF_RETURN_FAST, /* Sheaf return reattached spare sheaf */
389 SHEAF_RETURN_SLOW, /* Sheaf return could not reattach spare */
390 NR_SLUB_STAT_ITEMS
391};
392
393#ifdef CONFIG_SLUB_STATS
394struct kmem_cache_stats {
395 unsigned int stat[NR_SLUB_STAT_ITEMS];
396};
397#endif
398
399static inline void stat(const struct kmem_cache *s, enum stat_item si)
400{
401#ifdef CONFIG_SLUB_STATS
402 /*
403 * The rmw is racy on a preemptible kernel but this is acceptable, so
404 * avoid this_cpu_add()'s irq-disable overhead.
405 */
406 raw_cpu_inc(s->cpu_stats->stat[si]);
407#endif
408}
409
410static inline
411void stat_add(const struct kmem_cache *s, enum stat_item si, int v)
412{
413#ifdef CONFIG_SLUB_STATS
414 raw_cpu_add(s->cpu_stats->stat[si], v);
415#endif
416}
417
418#define MAX_FULL_SHEAVES 10
419#define MAX_EMPTY_SHEAVES 10
420
421struct node_barn {
422 spinlock_t lock;
423 struct list_head sheaves_full;
424 struct list_head sheaves_empty;
425 unsigned int nr_full;
426 unsigned int nr_empty;
427};
428
429struct slab_sheaf {
430 union {
431 struct rcu_head rcu_head;
432 struct list_head barn_list;
433 /* only used to defer call_rcu() in unknown context */
434 struct llist_node llnode;
435 /* only used for prefilled sheafs */
436 struct {
437 unsigned int capacity;
438 bool pfmemalloc;
439 };
440 };
441 struct kmem_cache *cache;
442 unsigned int size;
443 int node; /* only used for rcu_sheaf */
444 void *objects[];
445};
446
447struct slub_percpu_sheaves {
448 local_trylock_t lock;
449 struct slab_sheaf *main; /* never NULL when unlocked */
450 struct slab_sheaf *spare; /* empty or full, may be NULL */
451 struct slab_sheaf *rcu_free; /* for batching kfree_rcu() */
452};
453
454/*
455 * The slab lists for all objects.
456 */
457struct kmem_cache_node {
458 spinlock_t list_lock;
459 unsigned long nr_partial;
460 struct list_head partial;
461#ifdef CONFIG_SLUB_DEBUG
462 atomic_long_t nr_slabs;
463 atomic_long_t total_objects;
464 struct list_head full;
465#endif
466};
467
468static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
469{
470 return s->per_node[node].node;
471}
472
473static inline struct node_barn *get_barn_node(struct kmem_cache *s, int node)
474{
475 return s->per_node[node].barn;
476}
477
478/*
479 * Get the barn of the current cpu's NUMA node. It may be a memoryless node.
480 */
481static inline struct node_barn *get_barn(struct kmem_cache *s)
482{
483 return get_barn_node(s, numa_node_id());
484}
485
486/*
487 * Iterator over all nodes. The body will be executed for each node that has
488 * a kmem_cache_node structure allocated (which is true for all online nodes)
489 */
490#define for_each_kmem_cache_node(__s, __node, __n) \
491 for (__node = 0; __node < nr_node_ids; __node++) \
492 if ((__n = get_node(__s, __node)))
493
494/*
495 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
496 * Corresponds to node_state[N_MEMORY], but can temporarily
497 * differ during memory hotplug/hotremove operations.
498 * Protected by slab_mutex.
499 */
500static nodemask_t slab_nodes;
501
502/*
503 * Similar to slab_nodes but for where we have node_barn allocated.
504 * Corresponds to N_ONLINE nodes.
505 */
506static nodemask_t slab_barn_nodes;
507
508/*
509 * Workqueue used for flushing cpu and kfree_rcu sheaves.
510 */
511static struct workqueue_struct *flushwq;
512
513struct slub_flush_work {
514 struct work_struct work;
515 struct kmem_cache *s;
516 bool skip;
517};
518
519static DEFINE_MUTEX(flush_lock);
520static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);
521
522/********************************************************************
523 * Core slab cache functions
524 *******************************************************************/
525
526/*
527 * Returns freelist pointer (ptr). With hardening, this is obfuscated
528 * with an XOR of the address where the pointer is held and a per-cache
529 * random number.
530 */
531static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
532 void *ptr, unsigned long ptr_addr)
533{
534 unsigned long encoded;
535
536#ifdef CONFIG_SLAB_FREELIST_HARDENED
537 encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr);
538#else
539 encoded = (unsigned long)ptr;
540#endif
541 return (freeptr_t){.v = encoded};
542}
543
544static inline void *freelist_ptr_decode(const struct kmem_cache *s,
545 freeptr_t ptr, unsigned long ptr_addr)
546{
547 void *decoded;
548
549#ifdef CONFIG_SLAB_FREELIST_HARDENED
550 decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr));
551#else
552 decoded = (void *)ptr.v;
553#endif
554 return decoded;
555}
556
557static inline void *get_freepointer(struct kmem_cache *s, void *object)
558{
559 unsigned long ptr_addr;
560 freeptr_t p;
561
562 object = kasan_reset_tag(object);
563 ptr_addr = (unsigned long)object + s->offset;
564 p = *(freeptr_t *)(ptr_addr);
565 return freelist_ptr_decode(s, p, ptr_addr);
566}
567
568static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
569{
570 unsigned long freeptr_addr = (unsigned long)object + s->offset;
571
572#ifdef CONFIG_SLAB_FREELIST_HARDENED
573 BUG_ON(object == fp); /* naive detection of double free or corruption */
574#endif
575
576 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
577 *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);
578}
579
580/*
581 * See comment in calculate_sizes().
582 */
583static inline bool freeptr_outside_object(struct kmem_cache *s)
584{
585 return s->offset >= s->inuse;
586}
587
588/*
589 * Return offset of the end of info block which is inuse + free pointer if
590 * not overlapping with object.
591 */
592static inline unsigned int get_info_end(struct kmem_cache *s)
593{
594 if (freeptr_outside_object(s))
595 return s->inuse + sizeof(void *);
596 else
597 return s->inuse;
598}
599
600/* Loop over all objects in a slab */
601#define for_each_object(__p, __s, __addr, __objects) \
602 for (__p = fixup_red_left(__s, __addr); \
603 __p < (__addr) + (__objects) * (__s)->size; \
604 __p += (__s)->size)
605
606static inline unsigned int order_objects(unsigned int order, unsigned int size)
607{
608 return ((unsigned int)PAGE_SIZE << order) / size;
609}
610
611static inline struct kmem_cache_order_objects oo_make(unsigned int order,
612 unsigned int size)
613{
614 struct kmem_cache_order_objects x = {
615 (order << OO_SHIFT) + order_objects(order, size)
616 };
617
618 return x;
619}
620
621static inline unsigned int oo_order(struct kmem_cache_order_objects x)
622{
623 return x.x >> OO_SHIFT;
624}
625
626static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
627{
628 return x.x & OO_MASK;
629}
630
631/*
632 * If network-based swap is enabled, slub must keep track of whether memory
633 * were allocated from pfmemalloc reserves.
634 */
635static inline bool slab_test_pfmemalloc(const struct slab *slab)
636{
637 return test_bit(SL_pfmemalloc, &slab->flags.f);
638}
639
640static inline void slab_set_pfmemalloc(struct slab *slab)
641{
642 set_bit(SL_pfmemalloc, &slab->flags.f);
643}
644
645static inline void __slab_clear_pfmemalloc(struct slab *slab)
646{
647 __clear_bit(SL_pfmemalloc, &slab->flags.f);
648}
649
650/*
651 * Per slab locking using the pagelock
652 */
653static __always_inline void slab_lock(struct slab *slab)
654{
655 bit_spin_lock(SL_locked, &slab->flags.f);
656}
657
658static __always_inline void slab_unlock(struct slab *slab)
659{
660 bit_spin_unlock(SL_locked, &slab->flags.f);
661}
662
663static inline bool
664__update_freelist_fast(struct slab *slab, struct freelist_counters *old,
665 struct freelist_counters *new)
666{
667#ifdef system_has_freelist_aba
668 return try_cmpxchg_freelist(&slab->freelist_counters,
669 &old->freelist_counters,
670 new->freelist_counters);
671#else
672 return false;
673#endif
674}
675
676static inline bool
677__update_freelist_slow(struct slab *slab, struct freelist_counters *old,
678 struct freelist_counters *new)
679{
680 bool ret = false;
681
682 slab_lock(slab);
683 if (slab->freelist == old->freelist &&
684 slab->counters == old->counters) {
685 slab->freelist = new->freelist;
686 /* prevent tearing for the read in get_partial_node_bulk() */
687 WRITE_ONCE(slab->counters, new->counters);
688 ret = true;
689 }
690 slab_unlock(slab);
691
692 return ret;
693}
694
695/*
696 * Interrupts must be disabled (for the fallback code to work right), typically
697 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is
698 * part of bit_spin_lock(), is sufficient because the policy is not to allow any
699 * allocation/ free operation in hardirq context. Therefore nothing can
700 * interrupt the operation.
701 */
702static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
703 struct freelist_counters *old, struct freelist_counters *new, const char *n)
704{
705 bool ret;
706
707 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
708 lockdep_assert_irqs_disabled();
709
710 if (s->flags & __CMPXCHG_DOUBLE)
711 ret = __update_freelist_fast(slab, old, new);
712 else
713 ret = __update_freelist_slow(slab, old, new);
714
715 if (likely(ret))
716 return true;
717
718 cpu_relax();
719 stat(s, CMPXCHG_DOUBLE_FAIL);
720
721#ifdef SLUB_DEBUG_CMPXCHG
722 pr_info("%s %s: cmpxchg double redo ", n, s->name);
723#endif
724
725 return false;
726}
727
728static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
729 struct freelist_counters *old, struct freelist_counters *new, const char *n)
730{
731 bool ret;
732
733 if (s->flags & __CMPXCHG_DOUBLE) {
734 ret = __update_freelist_fast(slab, old, new);
735 } else {
736 unsigned long flags;
737
738 local_irq_save(flags);
739 ret = __update_freelist_slow(slab, old, new);
740 local_irq_restore(flags);
741 }
742 if (likely(ret))
743 return true;
744
745 cpu_relax();
746 stat(s, CMPXCHG_DOUBLE_FAIL);
747
748#ifdef SLUB_DEBUG_CMPXCHG
749 pr_info("%s %s: cmpxchg double redo ", n, s->name);
750#endif
751
752 return false;
753}
754
755/*
756 * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API
757 * family will round up the real request size to these fixed ones, so
758 * there could be an extra area than what is requested. Save the original
759 * request size in the meta data area, for better debug and sanity check.
760 */
761static inline void set_orig_size(struct kmem_cache *s,
762 void *object, unsigned long orig_size)
763{
764 void *p = kasan_reset_tag(object);
765
766 if (!slub_debug_orig_size(s))
767 return;
768
769 p += get_info_end(s);
770 p += sizeof(struct track) * 2;
771
772 *(unsigned long *)p = orig_size;
773}
774
775static inline unsigned long get_orig_size(struct kmem_cache *s, void *object)
776{
777 void *p = kasan_reset_tag(object);
778
779 if (is_kfence_address(object))
780 return kfence_ksize(object);
781
782 if (!slub_debug_orig_size(s))
783 return s->object_size;
784
785 p += get_info_end(s);
786 p += sizeof(struct track) * 2;
787
788 return *(unsigned long *)p;
789}
790
791#ifdef CONFIG_SLAB_OBJ_EXT
792
793/*
794 * Check if memory cgroup or memory allocation profiling is enabled.
795 * If enabled, SLUB tries to reduce memory overhead of accounting
796 * slab objects. If neither is enabled when this function is called,
797 * the optimization is simply skipped to avoid affecting caches that do not
798 * need slabobj_ext metadata.
799 *
800 * However, this may disable optimization when memory cgroup or memory
801 * allocation profiling is used, but slabs are created too early
802 * even before those subsystems are initialized.
803 */
804static inline bool need_slab_obj_exts(struct kmem_cache *s)
805{
806 if (s->flags & SLAB_NO_OBJ_EXT)
807 return false;
808
809 if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
810 return true;
811
812 if (mem_alloc_profiling_enabled())
813 return true;
814
815 return false;
816}
817
818static inline unsigned int obj_exts_size_in_slab(struct slab *slab)
819{
820 return slab_obj_ext_size(slab) * slab->objects;
821}
822
823static inline unsigned long obj_exts_offset_in_slab(struct kmem_cache *s,
824 struct slab *slab)
825{
826 unsigned long objext_offset;
827
828 objext_offset = s->size * slab->objects;
829 objext_offset = ALIGN(objext_offset, sizeof(struct slabobj_ext));
830 return objext_offset;
831}
832
833static inline bool obj_exts_fit_within_slab_leftover(struct kmem_cache *s,
834 struct slab *slab)
835{
836 unsigned long objext_offset = obj_exts_offset_in_slab(s, slab);
837 unsigned long objext_size = obj_exts_size_in_slab(slab);
838
839 return objext_offset + objext_size <= slab_size(slab);
840}
841
842static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)
843{
844 unsigned long obj_exts;
845 unsigned long start;
846 unsigned long end;
847
848 obj_exts = slab_obj_exts(slab);
849 if (!obj_exts)
850 return false;
851
852 start = (unsigned long)slab_address(slab);
853 end = start + slab_size(slab);
854 return (obj_exts >= start) && (obj_exts < end);
855}
856#else
857static inline bool need_slab_obj_exts(struct kmem_cache *s)
858{
859 return false;
860}
861
862static inline unsigned int obj_exts_size_in_slab(struct slab *slab)
863{
864 return 0;
865}
866
867static inline unsigned long obj_exts_offset_in_slab(struct kmem_cache *s,
868 struct slab *slab)
869{
870 return 0;
871}
872
873static inline bool obj_exts_fit_within_slab_leftover(struct kmem_cache *s,
874 struct slab *slab)
875{
876 return false;
877}
878
879static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)
880{
881 return false;
882}
883
884#endif
885
886#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
887static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
888{
889 unsigned int offset = get_info_end(s);
890
891 if (kmem_cache_debug_flags(s, SLAB_STORE_USER))
892 offset += sizeof(struct track) * 2;
893
894 if (slub_debug_orig_size(s))
895 offset += sizeof(unsigned long);
896
897 offset += kasan_metadata_size(s, false);
898
899 return offset;
900}
901
902static inline void slab_set_obj_exts_in_object(struct slab *slab)
903{
904 slab->obj_exts_in_object = 1;
905}
906#else
907static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
908{
909 return 0;
910}
911
912static inline void slab_set_obj_exts_in_object(struct slab *slab)
913{
914}
915#endif
916
917/*
918 * A no-op function used to attach kprobe handlers in slub_kunit tests.
919 * The barrier is needed to prevent the compiler from optimizing out callsites.
920 */
921#if defined(CONFIG_DEBUG_VM) || defined(CONFIG_PROVE_LOCKING)
922static noinline void slab_attach_kprobe_locked(void)
923{
924 barrier();
925}
926#else
927static inline void slab_attach_kprobe_locked(void) { }
928#endif
929
930#define slab_lockdep_assert_held(lock) do { \
931 lockdep_assert_held(lock); \
932 slab_attach_kprobe_locked(); \
933} while (0)
934
935#ifdef CONFIG_SLUB_DEBUG
936
937/*
938 * For debugging context when we want to check if the struct slab pointer
939 * appears to be valid.
940 */
941static inline bool validate_slab_ptr(struct slab *slab)
942{
943 return PageSlab(slab_page(slab));
944}
945
946static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
947static DEFINE_SPINLOCK(object_map_lock);
948
949static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
950 struct slab *slab)
951{
952 void *addr = slab_address(slab);
953 void *p;
954
955 bitmap_zero(obj_map, slab->objects);
956
957 for (p = slab->freelist; p; p = get_freepointer(s, p))
958 set_bit(__obj_to_index(s, addr, p), obj_map);
959}
960
961#if IS_ENABLED(CONFIG_KUNIT)
962static bool slab_add_kunit_errors(void)
963{
964 struct kunit_resource *resource;
965
966 if (!kunit_get_current_test())
967 return false;
968
969 resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
970 if (!resource)
971 return false;
972
973 (*(int *)resource->data)++;
974 kunit_put_resource(resource);
975 return true;
976}
977
978bool slab_in_kunit_test(void)
979{
980 struct kunit_resource *resource;
981
982 if (!kunit_get_current_test())
983 return false;
984
985 resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
986 if (!resource)
987 return false;
988
989 kunit_put_resource(resource);
990 return true;
991}
992#else
993static inline bool slab_add_kunit_errors(void) { return false; }
994#endif
995
996static inline unsigned int size_from_object(struct kmem_cache *s)
997{
998 if (s->flags & SLAB_RED_ZONE)
999 return s->size - s->red_left_pad;
1000
1001 return s->size;
1002}
1003
1004static inline void *restore_red_left(struct kmem_cache *s, void *p)
1005{
1006 if (s->flags & SLAB_RED_ZONE)
1007 p -= s->red_left_pad;
1008
1009 return p;
1010}
1011
1012/*
1013 * Debug settings:
1014 */
1015#if defined(CONFIG_SLUB_DEBUG_ON)
1016static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
1017#else
1018static slab_flags_t slub_debug;
1019#endif
1020
1021static const char *slub_debug_string __ro_after_init;
1022static int disable_higher_order_debug;
1023
1024/*
1025 * Object debugging
1026 */
1027
1028/* Verify that a pointer has an address that is valid within a slab page */
1029static inline int check_valid_pointer(struct kmem_cache *s,
1030 struct slab *slab, void *object)
1031{
1032 void *base;
1033
1034 if (!object)
1035 return 1;
1036
1037 base = slab_address(slab);
1038 object = kasan_reset_tag(object);
1039 object = restore_red_left(s, object);
1040 if (object < base || object >= base + slab->objects * s->size ||
1041 (object - base) % s->size) {
1042 return 0;
1043 }
1044
1045 return 1;
1046}
1047
1048static void print_section(char *level, char *text, u8 *addr,
1049 unsigned int length)
1050{
1051 metadata_access_enable();
1052 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
1053 16, 1, kasan_reset_tag((void *)addr), length, 1);
1054 metadata_access_disable();
1055}
1056
1057static struct track *get_track(struct kmem_cache *s, void *object,
1058 enum track_item alloc)
1059{
1060 struct track *p;
1061
1062 p = object + get_info_end(s);
1063
1064 return kasan_reset_tag(p + alloc);
1065}
1066
1067#ifdef CONFIG_STACKDEPOT
1068static noinline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags)
1069{
1070 depot_stack_handle_t handle;
1071 unsigned long entries[TRACK_ADDRS_COUNT];
1072 unsigned int nr_entries;
1073
1074 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
1075 handle = stack_depot_save(entries, nr_entries, gfp_flags);
1076
1077 return handle;
1078}
1079#else
1080static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags)
1081{
1082 return 0;
1083}
1084#endif
1085
1086static void set_track_update(struct kmem_cache *s, void *object,
1087 enum track_item alloc, unsigned long addr,
1088 depot_stack_handle_t handle)
1089{
1090 struct track *p = get_track(s, object, alloc);
1091
1092#ifdef CONFIG_STACKDEPOT
1093 p->handle = handle;
1094#endif
1095 p->addr = addr;
1096 p->cpu = raw_smp_processor_id();
1097 p->pid = current->pid;
1098 p->when = jiffies;
1099}
1100
1101static __always_inline void set_track(struct kmem_cache *s, void *object,
1102 enum track_item alloc, unsigned long addr, gfp_t gfp_flags)
1103{
1104 depot_stack_handle_t handle = set_track_prepare(gfp_flags);
1105
1106 set_track_update(s, object, alloc, addr, handle);
1107}
1108
1109static void init_tracking(struct kmem_cache *s, void *object)
1110{
1111 struct track *p;
1112
1113 if (!(s->flags & SLAB_STORE_USER))
1114 return;
1115
1116 p = get_track(s, object, TRACK_ALLOC);
1117 memset(p, 0, 2*sizeof(struct track));
1118}
1119
1120static void print_track(const char *s, struct track *t, unsigned long pr_time)
1121{
1122 depot_stack_handle_t handle __maybe_unused;
1123
1124 if (!t->addr)
1125 return;
1126
1127 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
1128 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
1129#ifdef CONFIG_STACKDEPOT
1130 handle = READ_ONCE(t->handle);
1131 if (handle)
1132 stack_depot_print(handle);
1133 else
1134 pr_err("object allocation/free stack trace missing\n");
1135#endif
1136}
1137
1138void print_tracking(struct kmem_cache *s, void *object)
1139{
1140 unsigned long pr_time = jiffies;
1141 if (!(s->flags & SLAB_STORE_USER))
1142 return;
1143
1144 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
1145 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
1146}
1147
1148static void print_slab_info(const struct slab *slab)
1149{
1150 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
1151 slab, slab->objects, slab->inuse, slab->freelist,
1152 &slab->flags.f);
1153}
1154
1155void skip_orig_size_check(struct kmem_cache *s, const void *object)
1156{
1157 set_orig_size(s, (void *)object, s->object_size);
1158}
1159
1160static void __slab_bug(struct kmem_cache *s, const char *fmt, va_list argsp)
1161{
1162 struct va_format vaf;
1163 va_list args;
1164
1165 va_copy(args, argsp);
1166 vaf.fmt = fmt;
1167 vaf.va = &args;
1168 pr_err("=============================================================================\n");
1169 pr_err("BUG %s (%s): %pV\n", s ? s->name : "<unknown>", print_tainted(), &vaf);
1170 pr_err("-----------------------------------------------------------------------------\n\n");
1171 va_end(args);
1172}
1173
1174static void slab_bug(struct kmem_cache *s, const char *fmt, ...)
1175{
1176 va_list args;
1177
1178 va_start(args, fmt);
1179 __slab_bug(s, fmt, args);
1180 va_end(args);
1181}
1182
1183__printf(2, 3)
1184static void slab_fix(struct kmem_cache *s, const char *fmt, ...)
1185{
1186 struct va_format vaf;
1187 va_list args;
1188
1189 if (slab_add_kunit_errors())
1190 return;
1191
1192 va_start(args, fmt);
1193 vaf.fmt = fmt;
1194 vaf.va = &args;
1195 pr_err("FIX %s: %pV\n", s->name, &vaf);
1196 va_end(args);
1197}
1198
1199static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
1200{
1201 unsigned int off; /* Offset of last byte */
1202 u8 *addr = slab_address(slab);
1203
1204 print_tracking(s, p);
1205
1206 print_slab_info(slab);
1207
1208 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
1209 p, p - addr, get_freepointer(s, p));
1210
1211 if (s->flags & SLAB_RED_ZONE)
1212 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
1213 s->red_left_pad);
1214 else if (p > addr + 16)
1215 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
1216
1217 print_section(KERN_ERR, "Object ", p,
1218 min_t(unsigned int, s->object_size, PAGE_SIZE));
1219 if (s->flags & SLAB_RED_ZONE)
1220 print_section(KERN_ERR, "Redzone ", p + s->object_size,
1221 s->inuse - s->object_size);
1222
1223 off = get_info_end(s);
1224
1225 if (s->flags & SLAB_STORE_USER)
1226 off += 2 * sizeof(struct track);
1227
1228 if (slub_debug_orig_size(s))
1229 off += sizeof(unsigned long);
1230
1231 off += kasan_metadata_size(s, false);
1232
1233 if (obj_exts_in_object(slab))
1234 off += slab_obj_ext_size(slab);
1235
1236 if (off != size_from_object(s))
1237 /* Beginning of the filler is the free pointer */
1238 print_section(KERN_ERR, "Padding ", p + off,
1239 size_from_object(s) - off);
1240}
1241
1242static void object_err(struct kmem_cache *s, struct slab *slab,
1243 u8 *object, const char *reason)
1244{
1245 if (slab_add_kunit_errors())
1246 return;
1247
1248 slab_bug(s, reason);
1249 if (!object || !check_valid_pointer(s, slab, object)) {
1250 print_slab_info(slab);
1251 pr_err("Invalid pointer 0x%p\n", object);
1252 } else {
1253 print_trailer(s, slab, object);
1254 }
1255 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1256
1257 WARN_ON(1);
1258}
1259
1260static void __slab_err(struct slab *slab)
1261{
1262 if (slab_in_kunit_test())
1263 return;
1264
1265 print_slab_info(slab);
1266 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1267
1268 WARN_ON(1);
1269}
1270
1271static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
1272 const char *fmt, ...)
1273{
1274 va_list args;
1275
1276 if (slab_add_kunit_errors())
1277 return;
1278
1279 va_start(args, fmt);
1280 __slab_bug(s, fmt, args);
1281 va_end(args);
1282
1283 __slab_err(slab);
1284}
1285
1286static void init_object(struct kmem_cache *s, void *object, u8 val)
1287{
1288 u8 *p = kasan_reset_tag(object);
1289 unsigned int poison_size = s->object_size;
1290
1291 if (s->flags & SLAB_RED_ZONE) {
1292 /*
1293 * Here and below, avoid overwriting the KMSAN shadow. Keeping
1294 * the shadow makes it possible to distinguish uninit-value
1295 * from use-after-free.
1296 */
1297 memset_no_sanitize_memory(p - s->red_left_pad, val,
1298 s->red_left_pad);
1299
1300 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1301 /*
1302 * Redzone the extra allocated space by kmalloc than
1303 * requested, and the poison size will be limited to
1304 * the original request size accordingly.
1305 */
1306 poison_size = get_orig_size(s, object);
1307 }
1308 }
1309
1310 if (s->flags & __OBJECT_POISON) {
1311 memset_no_sanitize_memory(p, POISON_FREE, poison_size - 1);
1312 memset_no_sanitize_memory(p + poison_size - 1, POISON_END, 1);
1313 }
1314
1315 if (s->flags & SLAB_RED_ZONE)
1316 memset_no_sanitize_memory(p + poison_size, val,
1317 s->inuse - poison_size);
1318}
1319
1320static void restore_bytes(struct kmem_cache *s, const char *message, u8 data,
1321 void *from, void *to)
1322{
1323 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
1324 memset(from, data, to - from);
1325}
1326
1327#ifdef CONFIG_KMSAN
1328#define pad_check_attributes noinline __no_kmsan_checks
1329#else
1330#define pad_check_attributes
1331#endif
1332
1333static pad_check_attributes int
1334check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
1335 u8 *object, const char *what, u8 *start, unsigned int value,
1336 unsigned int bytes, bool slab_obj_print)
1337{
1338 u8 *fault;
1339 u8 *end;
1340 u8 *addr = slab_address(slab);
1341
1342 metadata_access_enable();
1343 fault = memchr_inv(kasan_reset_tag(start), value, bytes);
1344 metadata_access_disable();
1345 if (!fault)
1346 return 1;
1347
1348 end = start + bytes;
1349 while (end > fault && end[-1] == value)
1350 end--;
1351
1352 if (slab_add_kunit_errors())
1353 goto skip_bug_print;
1354
1355 pr_err("[%s overwritten] 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
1356 what, fault, end - 1, fault - addr, fault[0], value);
1357
1358 if (slab_obj_print)
1359 object_err(s, slab, object, "Object corrupt");
1360
1361skip_bug_print:
1362 restore_bytes(s, what, value, fault, end);
1363 return 0;
1364}
1365
1366/*
1367 * Object field layout:
1368 *
1369 * [Left redzone padding] (if SLAB_RED_ZONE)
1370 * - Field size: s->red_left_pad
1371 * - Immediately precedes each object when SLAB_RED_ZONE is set.
1372 * - Filled with 0xbb (SLUB_RED_INACTIVE) for inactive objects and
1373 * 0xcc (SLUB_RED_ACTIVE) for objects in use when SLAB_RED_ZONE.
1374 *
1375 * [Object bytes] (object address starts here)
1376 * - Field size: s->object_size
1377 * - Object payload bytes.
1378 * - If the freepointer may overlap the object, it is stored inside
1379 * the object (typically near the middle).
1380 * - Poisoning uses 0x6b (POISON_FREE) and the last byte is
1381 * 0xa5 (POISON_END) when __OBJECT_POISON is enabled.
1382 *
1383 * [Word-align padding] (right redzone when SLAB_RED_ZONE is set)
1384 * - Field size: s->inuse - s->object_size
1385 * - If redzoning is enabled and ALIGN(size, sizeof(void *)) adds no
1386 * padding, explicitly extend by one word so the right redzone is
1387 * non-empty.
1388 * - Filled with 0xbb (SLUB_RED_INACTIVE) for inactive objects and
1389 * 0xcc (SLUB_RED_ACTIVE) for objects in use when SLAB_RED_ZONE.
1390 *
1391 * [Metadata starts at object + s->inuse]
1392 * - A. freelist pointer (if freeptr_outside_object)
1393 * - B. alloc tracking (SLAB_STORE_USER)
1394 * - C. free tracking (SLAB_STORE_USER)
1395 * - D. original request size (SLAB_KMALLOC && SLAB_STORE_USER)
1396 * - E. KASAN metadata (if enabled)
1397 *
1398 * [Mandatory padding] (if CONFIG_SLUB_DEBUG && SLAB_RED_ZONE)
1399 * - One mandatory debug word to guarantee a minimum poisoned gap
1400 * between metadata and the next object, independent of alignment.
1401 * - Filled with 0x5a (POISON_INUSE) when SLAB_POISON is set.
1402 * [Final alignment padding]
1403 * - Bytes added by ALIGN(size, s->align) to reach s->size.
1404 * - When the padding is large enough, it can be used to store
1405 * struct slabobj_ext for accounting metadata (obj_exts_in_object()).
1406 * - The remaining bytes (if any) are filled with 0x5a (POISON_INUSE)
1407 * when SLAB_POISON is set.
1408 *
1409 * Notes:
1410 * - Redzones are filled by init_object() with SLUB_RED_ACTIVE/INACTIVE.
1411 * - Object contents are poisoned with POISON_FREE/END when __OBJECT_POISON.
1412 * - The trailing padding is pre-filled with POISON_INUSE by
1413 * setup_slab_debug() when SLAB_POISON is set, and is validated by
1414 * check_pad_bytes().
1415 * - The first object pointer is slab_address(slab) +
1416 * (s->red_left_pad if redzoning); subsequent objects are reached by
1417 * adding s->size each time.
1418 *
1419 * If a slab cache flag relies on specific metadata to exist at a fixed
1420 * offset, the flag must be included in SLAB_NEVER_MERGE to prevent merging.
1421 * Otherwise, the cache would misbehave as s->object_size and s->inuse are
1422 * adjusted during cache merging (see __kmem_cache_alias()).
1423 */
1424static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
1425{
1426 unsigned long off = get_info_end(s); /* The end of info */
1427
1428 if (s->flags & SLAB_STORE_USER) {
1429 /* We also have user information there */
1430 off += 2 * sizeof(struct track);
1431
1432 if (s->flags & SLAB_KMALLOC)
1433 off += sizeof(unsigned long);
1434 }
1435
1436 off += kasan_metadata_size(s, false);
1437
1438 if (obj_exts_in_object(slab))
1439 off += slab_obj_ext_size(slab);
1440
1441 if (size_from_object(s) == off)
1442 return 1;
1443
1444 return check_bytes_and_report(s, slab, p, "Object padding",
1445 p + off, POISON_INUSE, size_from_object(s) - off, true);
1446}
1447
1448/* Check the pad bytes at the end of a slab page */
1449static pad_check_attributes void
1450slab_pad_check(struct kmem_cache *s, struct slab *slab)
1451{
1452 u8 *start;
1453 u8 *fault;
1454 u8 *end;
1455 u8 *pad;
1456 int length;
1457 int remainder;
1458
1459 if (!(s->flags & SLAB_POISON))
1460 return;
1461
1462 start = slab_address(slab);
1463 length = slab_size(slab);
1464 end = start + length;
1465
1466 if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(slab)) {
1467 remainder = length;
1468 remainder -= obj_exts_offset_in_slab(s, slab);
1469 remainder -= obj_exts_size_in_slab(slab);
1470 } else {
1471 remainder = length % s->size;
1472 }
1473
1474 if (!remainder)
1475 return;
1476
1477 pad = end - remainder;
1478 metadata_access_enable();
1479 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
1480 metadata_access_disable();
1481 if (!fault)
1482 return;
1483 while (end > fault && end[-1] == POISON_INUSE)
1484 end--;
1485
1486 slab_bug(s, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1487 fault, end - 1, fault - start);
1488 print_section(KERN_ERR, "Padding ", pad, remainder);
1489 __slab_err(slab);
1490
1491 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
1492}
1493
1494static int check_object(struct kmem_cache *s, struct slab *slab,
1495 void *object, u8 val)
1496{
1497 u8 *p = object;
1498 u8 *endobject = object + s->object_size;
1499 unsigned int orig_size, kasan_meta_size;
1500 int ret = 1;
1501
1502 if (s->flags & SLAB_RED_ZONE) {
1503 if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1504 object - s->red_left_pad, val, s->red_left_pad, ret))
1505 ret = 0;
1506
1507 if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1508 endobject, val, s->inuse - s->object_size, ret))
1509 ret = 0;
1510
1511 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1512 orig_size = get_orig_size(s, object);
1513
1514 if (s->object_size > orig_size &&
1515 !check_bytes_and_report(s, slab, object,
1516 "kmalloc Redzone", p + orig_size,
1517 val, s->object_size - orig_size, ret)) {
1518 ret = 0;
1519 }
1520 }
1521 } else {
1522 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
1523 if (!check_bytes_and_report(s, slab, p, "Alignment padding",
1524 endobject, POISON_INUSE,
1525 s->inuse - s->object_size, ret))
1526 ret = 0;
1527 }
1528 }
1529
1530 if (s->flags & SLAB_POISON) {
1531 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) {
1532 /*
1533 * KASAN can save its free meta data inside of the
1534 * object at offset 0. Thus, skip checking the part of
1535 * the redzone that overlaps with the meta data.
1536 */
1537 kasan_meta_size = kasan_metadata_size(s, true);
1538 if (kasan_meta_size < s->object_size - 1 &&
1539 !check_bytes_and_report(s, slab, p, "Poison",
1540 p + kasan_meta_size, POISON_FREE,
1541 s->object_size - kasan_meta_size - 1, ret))
1542 ret = 0;
1543 if (kasan_meta_size < s->object_size &&
1544 !check_bytes_and_report(s, slab, p, "End Poison",
1545 p + s->object_size - 1, POISON_END, 1, ret))
1546 ret = 0;
1547 }
1548 /*
1549 * check_pad_bytes cleans up on its own.
1550 */
1551 if (!check_pad_bytes(s, slab, p))
1552 ret = 0;
1553 }
1554
1555 /*
1556 * Cannot check freepointer while object is allocated if
1557 * object and freepointer overlap.
1558 */
1559 if ((freeptr_outside_object(s) || val != SLUB_RED_ACTIVE) &&
1560 !check_valid_pointer(s, slab, get_freepointer(s, p))) {
1561 object_err(s, slab, p, "Freepointer corrupt");
1562 /*
1563 * No choice but to zap it and thus lose the remainder
1564 * of the free objects in this slab. May cause
1565 * another error because the object count is now wrong.
1566 */
1567 set_freepointer(s, p, NULL);
1568 ret = 0;
1569 }
1570
1571 return ret;
1572}
1573
1574/*
1575 * Checks if the slab state looks sane. Assumes the struct slab pointer
1576 * was either obtained in a way that ensures it's valid, or validated
1577 * by validate_slab_ptr()
1578 */
1579static int check_slab(struct kmem_cache *s, struct slab *slab)
1580{
1581 int maxobj;
1582
1583 maxobj = order_objects(slab_order(slab), s->size);
1584 if (slab->objects > maxobj) {
1585 slab_err(s, slab, "objects %u > max %u",
1586 slab->objects, maxobj);
1587 return 0;
1588 }
1589 if (slab->inuse > slab->objects) {
1590 slab_err(s, slab, "inuse %u > max %u",
1591 slab->inuse, slab->objects);
1592 return 0;
1593 }
1594 if (slab->frozen) {
1595 slab_err(s, slab, "Slab disabled since SLUB metadata consistency check failed");
1596 return 0;
1597 }
1598
1599 /* Slab_pad_check fixes things up after itself */
1600 slab_pad_check(s, slab);
1601 return 1;
1602}
1603
1604/*
1605 * Determine if a certain object in a slab is on the freelist. Must hold the
1606 * slab lock to guarantee that the chains are in a consistent state.
1607 */
1608static bool on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
1609{
1610 int nr = 0;
1611 void *fp;
1612 void *object = NULL;
1613 int max_objects;
1614
1615 fp = slab->freelist;
1616 while (fp && nr <= slab->objects) {
1617 if (fp == search)
1618 return true;
1619 if (!check_valid_pointer(s, slab, fp)) {
1620 if (object) {
1621 object_err(s, slab, object,
1622 "Freechain corrupt");
1623 set_freepointer(s, object, NULL);
1624 break;
1625 } else {
1626 slab_err(s, slab, "Freepointer corrupt");
1627 slab->freelist = NULL;
1628 slab->inuse = slab->objects;
1629 slab_fix(s, "Freelist cleared");
1630 return false;
1631 }
1632 }
1633 object = fp;
1634 fp = get_freepointer(s, object);
1635 nr++;
1636 }
1637
1638 if (nr > slab->objects) {
1639 slab_err(s, slab, "Freelist cycle detected");
1640 slab->freelist = NULL;
1641 slab->inuse = slab->objects;
1642 slab_fix(s, "Freelist cleared");
1643 return false;
1644 }
1645
1646 max_objects = order_objects(slab_order(slab), s->size);
1647 if (max_objects > MAX_OBJS_PER_PAGE)
1648 max_objects = MAX_OBJS_PER_PAGE;
1649
1650 if (slab->objects != max_objects) {
1651 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1652 slab->objects, max_objects);
1653 slab->objects = max_objects;
1654 slab_fix(s, "Number of objects adjusted");
1655 }
1656 if (slab->inuse != slab->objects - nr) {
1657 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1658 slab->inuse, slab->objects - nr);
1659 slab->inuse = slab->objects - nr;
1660 slab_fix(s, "Object count adjusted");
1661 }
1662 return search == NULL;
1663}
1664
1665static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1666 int alloc)
1667{
1668 if (s->flags & SLAB_TRACE) {
1669 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
1670 s->name,
1671 alloc ? "alloc" : "free",
1672 object, slab->inuse,
1673 slab->freelist);
1674
1675 if (!alloc)
1676 print_section(KERN_INFO, "Object ", (void *)object,
1677 s->object_size);
1678
1679 dump_stack();
1680 }
1681}
1682
1683/*
1684 * Tracking of fully allocated slabs for debugging purposes.
1685 */
1686static void add_full(struct kmem_cache *s,
1687 struct kmem_cache_node *n, struct slab *slab)
1688{
1689 if (!(s->flags & SLAB_STORE_USER))
1690 return;
1691
1692 slab_lockdep_assert_held(&n->list_lock);
1693 list_add(&slab->slab_list, &n->full);
1694}
1695
1696static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1697{
1698 if (!(s->flags & SLAB_STORE_USER))
1699 return;
1700
1701 slab_lockdep_assert_held(&n->list_lock);
1702 list_del(&slab->slab_list);
1703}
1704
1705static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1706{
1707 return atomic_long_read(&n->nr_slabs);
1708}
1709
1710static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1711{
1712 struct kmem_cache_node *n = get_node(s, node);
1713
1714 atomic_long_inc(&n->nr_slabs);
1715 atomic_long_add(objects, &n->total_objects);
1716}
1717static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1718{
1719 struct kmem_cache_node *n = get_node(s, node);
1720
1721 atomic_long_dec(&n->nr_slabs);
1722 atomic_long_sub(objects, &n->total_objects);
1723}
1724
1725/* Object debug checks for alloc/free paths */
1726static void setup_object_debug(struct kmem_cache *s, void *object)
1727{
1728 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
1729 return;
1730
1731 init_object(s, object, SLUB_RED_INACTIVE);
1732 init_tracking(s, object);
1733}
1734
1735static
1736void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1737{
1738 if (!kmem_cache_debug_flags(s, SLAB_POISON))
1739 return;
1740
1741 metadata_access_enable();
1742 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
1743 metadata_access_disable();
1744}
1745
1746static inline int alloc_consistency_checks(struct kmem_cache *s,
1747 struct slab *slab, void *object)
1748{
1749 if (!check_slab(s, slab))
1750 return 0;
1751
1752 if (!check_valid_pointer(s, slab, object)) {
1753 object_err(s, slab, object, "Freelist Pointer check fails");
1754 return 0;
1755 }
1756
1757 if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1758 return 0;
1759
1760 return 1;
1761}
1762
1763static noinline bool alloc_debug_processing(struct kmem_cache *s,
1764 struct slab *slab, void *object, int orig_size)
1765{
1766 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1767 if (!alloc_consistency_checks(s, slab, object))
1768 goto bad;
1769 }
1770
1771 /* Success. Perform special debug activities for allocs */
1772 trace(s, slab, object, 1);
1773 set_orig_size(s, object, orig_size);
1774 init_object(s, object, SLUB_RED_ACTIVE);
1775 return true;
1776
1777bad:
1778 /*
1779 * Let's do the best we can to avoid issues in the future. Marking all
1780 * objects as used avoids touching the remaining objects.
1781 */
1782 slab_fix(s, "Marking all objects used");
1783 slab->inuse = slab->objects;
1784 slab->freelist = NULL;
1785 slab->frozen = 1; /* mark consistency-failed slab as frozen */
1786
1787 return false;
1788}
1789
1790static inline int free_consistency_checks(struct kmem_cache *s,
1791 struct slab *slab, void *object, unsigned long addr)
1792{
1793 if (!check_valid_pointer(s, slab, object)) {
1794 slab_err(s, slab, "Invalid object pointer 0x%p", object);
1795 return 0;
1796 }
1797
1798 if (on_freelist(s, slab, object)) {
1799 object_err(s, slab, object, "Object already free");
1800 return 0;
1801 }
1802
1803 if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1804 return 0;
1805
1806 if (unlikely(s != slab->slab_cache)) {
1807 if (!slab->slab_cache) {
1808 slab_err(NULL, slab, "No slab cache for object 0x%p",
1809 object);
1810 } else {
1811 object_err(s, slab, object,
1812 "page slab pointer corrupt.");
1813 }
1814 return 0;
1815 }
1816 return 1;
1817}
1818
1819/*
1820 * Parse a block of slab_debug options. Blocks are delimited by ';'
1821 *
1822 * @str: start of block
1823 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
1824 * @slabs: return start of list of slabs, or NULL when there's no list
1825 * @init: assume this is initial parsing and not per-kmem-create parsing
1826 *
1827 * returns the start of next block if there's any, or NULL
1828 */
1829static const char *
1830parse_slub_debug_flags(const char *str, slab_flags_t *flags, const char **slabs, bool init)
1831{
1832 bool higher_order_disable = false;
1833
1834 /* Skip any completely empty blocks */
1835 while (*str && *str == ';')
1836 str++;
1837
1838 if (*str == ',') {
1839 /*
1840 * No options but restriction on slabs. This means full
1841 * debugging for slabs matching a pattern.
1842 */
1843 *flags = DEBUG_DEFAULT_FLAGS;
1844 goto check_slabs;
1845 }
1846 *flags = 0;
1847
1848 /* Determine which debug features should be switched on */
1849 for (; *str && *str != ',' && *str != ';'; str++) {
1850 switch (tolower(*str)) {
1851 case '-':
1852 *flags = 0;
1853 break;
1854 case 'f':
1855 *flags |= SLAB_CONSISTENCY_CHECKS;
1856 break;
1857 case 'z':
1858 *flags |= SLAB_RED_ZONE;
1859 break;
1860 case 'p':
1861 *flags |= SLAB_POISON;
1862 break;
1863 case 'u':
1864 *flags |= SLAB_STORE_USER;
1865 break;
1866 case 't':
1867 *flags |= SLAB_TRACE;
1868 break;
1869 case 'a':
1870 *flags |= SLAB_FAILSLAB;
1871 break;
1872 case 'o':
1873 /*
1874 * Avoid enabling debugging on caches if its minimum
1875 * order would increase as a result.
1876 */
1877 higher_order_disable = true;
1878 break;
1879 default:
1880 if (init)
1881 pr_err("slab_debug option '%c' unknown. skipped\n", *str);
1882 }
1883 }
1884check_slabs:
1885 if (*str == ',')
1886 *slabs = ++str;
1887 else
1888 *slabs = NULL;
1889
1890 /* Skip over the slab list */
1891 while (*str && *str != ';')
1892 str++;
1893
1894 /* Skip any completely empty blocks */
1895 while (*str && *str == ';')
1896 str++;
1897
1898 if (init && higher_order_disable)
1899 disable_higher_order_debug = 1;
1900
1901 if (*str)
1902 return str;
1903 else
1904 return NULL;
1905}
1906
1907static int __init setup_slub_debug(const char *str, const struct kernel_param *kp)
1908{
1909 slab_flags_t flags;
1910 slab_flags_t global_flags;
1911 const char *saved_str;
1912 const char *slab_list;
1913 bool global_slub_debug_changed = false;
1914 bool slab_list_specified = false;
1915
1916 global_flags = DEBUG_DEFAULT_FLAGS;
1917 if (!str || !*str)
1918 /*
1919 * No options specified. Switch on full debugging.
1920 */
1921 goto out;
1922
1923 saved_str = str;
1924 while (str) {
1925 str = parse_slub_debug_flags(str, &flags, &slab_list, true);
1926
1927 if (!slab_list) {
1928 global_flags = flags;
1929 global_slub_debug_changed = true;
1930 } else {
1931 slab_list_specified = true;
1932 if (flags & SLAB_STORE_USER)
1933 stack_depot_request_early_init();
1934 }
1935 }
1936
1937 /*
1938 * For backwards compatibility, a single list of flags with list of
1939 * slabs means debugging is only changed for those slabs, so the global
1940 * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1941 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
1942 * long as there is no option specifying flags without a slab list.
1943 */
1944 if (slab_list_specified) {
1945 if (!global_slub_debug_changed)
1946 global_flags = slub_debug;
1947 slub_debug_string = saved_str;
1948 }
1949out:
1950 slub_debug = global_flags;
1951 if (slub_debug & SLAB_STORE_USER)
1952 stack_depot_request_early_init();
1953 if (slub_debug != 0 || slub_debug_string)
1954 static_branch_enable(&slub_debug_enabled);
1955 else
1956 static_branch_disable(&slub_debug_enabled);
1957 if ((static_branch_unlikely(&init_on_alloc) ||
1958 static_branch_unlikely(&init_on_free)) &&
1959 (slub_debug & SLAB_POISON))
1960 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
1961 return 0;
1962}
1963
1964static const struct kernel_param_ops param_ops_slab_debug __initconst = {
1965 .flags = KERNEL_PARAM_OPS_FL_NOARG,
1966 .set = setup_slub_debug,
1967};
1968__core_param_cb(slab_debug, &param_ops_slab_debug, NULL, 0);
1969__core_param_cb(slub_debug, &param_ops_slab_debug, NULL, 0);
1970
1971/*
1972 * kmem_cache_flags - apply debugging options to the cache
1973 * @flags: flags to set
1974 * @name: name of the cache
1975 *
1976 * Debug option(s) are applied to @flags. In addition to the debug
1977 * option(s), if a slab name (or multiple) is specified i.e.
1978 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1979 * then only the select slabs will receive the debug option(s).
1980 */
1981slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1982{
1983 const char *iter;
1984 size_t len;
1985 const char *next_block;
1986 slab_flags_t block_flags;
1987 slab_flags_t slub_debug_local = slub_debug;
1988
1989 if (flags & SLAB_NO_USER_FLAGS)
1990 return flags;
1991
1992 /*
1993 * If the slab cache is for debugging (e.g. kmemleak) then
1994 * don't store user (stack trace) information by default,
1995 * but let the user enable it via the command line below.
1996 */
1997 if (flags & SLAB_NOLEAKTRACE)
1998 slub_debug_local &= ~SLAB_STORE_USER;
1999
2000 len = strlen(name);
2001 next_block = slub_debug_string;
2002 /* Go through all blocks of debug options, see if any matches our slab's name */
2003 while (next_block) {
2004 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
2005 if (!iter)
2006 continue;
2007 /* Found a block that has a slab list, search it */
2008 while (*iter) {
2009 const char *end, *glob;
2010 size_t cmplen;
2011
2012 end = strchrnul(iter, ',');
2013 if (next_block && next_block < end)
2014 end = next_block - 1;
2015
2016 glob = strnchr(iter, end - iter, '*');
2017 if (glob)
2018 cmplen = glob - iter;
2019 else
2020 cmplen = max_t(size_t, len, (end - iter));
2021
2022 if (!strncmp(name, iter, cmplen)) {
2023 flags |= block_flags;
2024 return flags;
2025 }
2026
2027 if (!*end || *end == ';')
2028 break;
2029 iter = end + 1;
2030 }
2031 }
2032
2033 return flags | slub_debug_local;
2034}
2035#else /* !CONFIG_SLUB_DEBUG */
2036static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
2037static inline
2038void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
2039
2040static inline bool alloc_debug_processing(struct kmem_cache *s,
2041 struct slab *slab, void *object, int orig_size) { return true; }
2042
2043static inline bool free_debug_processing(struct kmem_cache *s,
2044 struct slab *slab, void *head, void *tail, int *bulk_cnt,
2045 unsigned long addr, depot_stack_handle_t handle) { return true; }
2046
2047static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
2048static inline int check_object(struct kmem_cache *s, struct slab *slab,
2049 void *object, u8 val) { return 1; }
2050static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) { return 0; }
2051static inline void set_track(struct kmem_cache *s, void *object,
2052 enum track_item alloc, unsigned long addr, gfp_t gfp_flags) {}
2053static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
2054 struct slab *slab) {}
2055static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
2056 struct slab *slab) {}
2057slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
2058{
2059 return flags;
2060}
2061#define slub_debug 0
2062
2063#define disable_higher_order_debug 0
2064
2065static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
2066 { return 0; }
2067static inline void inc_slabs_node(struct kmem_cache *s, int node,
2068 int objects) {}
2069static inline void dec_slabs_node(struct kmem_cache *s, int node,
2070 int objects) {}
2071#endif /* CONFIG_SLUB_DEBUG */
2072
2073/*
2074 * The allocated objcg pointers array or sheaf is not accounted directly.
2075 * Moreover, it should not come from DMA buffer and is not readily
2076 * reclaimable. Node restriction for the parent allocation also should
2077 * not apply to the slab's internal objects, as well as __GFP_COMP used
2078 * for new slab allocations.
2079 * So those GFP bits should be masked off.
2080 */
2081#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \
2082 __GFP_ACCOUNT | __GFP_NOFAIL | \
2083 __GFP_THISNODE | __GFP_COMP)
2084
2085#ifdef CONFIG_SLAB_OBJ_EXT
2086
2087#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
2088
2089static inline void mark_obj_codetag_empty(const void *obj)
2090{
2091 struct slab *obj_slab;
2092 unsigned long slab_exts;
2093
2094 if (!slab_obj_ext_has_codetag())
2095 return;
2096
2097 obj_slab = virt_to_slab(obj);
2098 slab_exts = slab_obj_exts(obj_slab);
2099 if (slab_exts) {
2100 struct slabobj_ext *ext;
2101 union codetag_ref *ref;
2102
2103 get_slab_obj_exts(slab_exts);
2104 ext = slab_obj_ext(obj_slab->slab_cache, obj_slab, slab_exts, obj);
2105 ref = slab_obj_ext_codetag_ref(obj_slab, ext);
2106
2107 if (unlikely(is_codetag_empty(ref))) {
2108 put_slab_obj_exts(slab_exts);
2109 return;
2110 }
2111
2112 /* codetag should be NULL here */
2113 WARN_ON(ref->ct);
2114 set_codetag_empty(ref);
2115 put_slab_obj_exts(slab_exts);
2116 }
2117}
2118
2119static inline bool mark_failed_objexts_alloc(struct slab *slab)
2120{
2121 return cmpxchg(&slab->obj_exts, 0, OBJEXTS_ALLOC_FAIL) == 0;
2122}
2123
2124static inline void handle_failed_objexts_alloc(struct slab *slab,
2125 unsigned long obj_exts, struct slabobj_ext *vec)
2126{
2127 unsigned int stride;
2128
2129 if (!slab_obj_ext_has_codetag())
2130 return;
2131
2132 /*
2133 * If vector previously failed to allocate then we have live
2134 * objects with no tag reference. Mark all references in this
2135 * vector as empty to avoid warnings later on.
2136 */
2137 if (obj_exts != OBJEXTS_ALLOC_FAIL)
2138 return;
2139
2140 stride = slab_obj_ext_size(slab) / sizeof(*vec);
2141
2142 for (unsigned int i = 0; i < slab->objects; i++) {
2143 union codetag_ref *ref = slab_obj_ext_codetag_ref(slab, vec);
2144
2145 set_codetag_empty(ref);
2146 vec += stride;
2147 }
2148}
2149
2150#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
2151
2152static inline void mark_obj_codetag_empty(const void *obj) {}
2153static inline bool mark_failed_objexts_alloc(struct slab *slab) { return false; }
2154static inline void handle_failed_objexts_alloc(struct slab *slab,
2155 unsigned long obj_exts, struct slabobj_ext *vec) {}
2156
2157#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
2158
2159static inline void init_slab_obj_exts(struct slab *slab)
2160{
2161 slab->obj_exts = 0;
2162}
2163
2164int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
2165 gfp_t gfp, unsigned int alloc_flags)
2166{
2167 const bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
2168 bool new_slab = alloc_flags & SLAB_ALLOC_NEW_SLAB;
2169 unsigned long new_exts;
2170 unsigned long old_exts;
2171 struct slabobj_ext *vec;
2172 size_t sz = slab_obj_ext_size(slab) * slab->objects;
2173
2174 gfp &= ~OBJCGS_CLEAR_MASK;
2175 /*
2176 * In most cases, obj_exts arrays are allocated from normal kmalloc.
2177 * However, normal kmalloc caches must allocate them from
2178 * KMALLOC_NO_OBJ_EXT caches to prevent recursion.
2179 */
2180 if (is_kmalloc_normal(s))
2181 alloc_flags |= SLAB_ALLOC_NO_OBJ_EXT;
2182
2183 alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
2184
2185 /* This will use kmalloc_nolock() if alloc_flags say so */
2186 vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab));
2187
2188 if (!vec) {
2189 /*
2190 * Try to mark vectors which failed to allocate.
2191 * If this operation fails, there may be a racing process
2192 * that has already completed the allocation.
2193 */
2194 if (!mark_failed_objexts_alloc(slab) &&
2195 slab_obj_exts(slab))
2196 return 0;
2197
2198 return -ENOMEM;
2199 }
2200
2201 if (IS_ENABLED(CONFIG_DEBUG_VM)) {
2202 struct kmem_cache *exts_cache;
2203 struct slab *exts_slab;
2204
2205 exts_slab = virt_to_slab(vec);
2206 if (exts_slab) {
2207 /*
2208 * The vector must be allocated from either normal or
2209 * KMALLOC_NO_OBJ_EXT kmalloc caches to avoid cycles.
2210 */
2211 exts_cache = exts_slab->slab_cache;
2212 WARN_ON_ONCE(!is_kmalloc_normal(exts_cache) &&
2213 !(exts_cache->flags & SLAB_NO_OBJ_EXT));
2214 }
2215 }
2216
2217 new_exts = (unsigned long)vec;
2218#ifdef CONFIG_MEMCG
2219 new_exts |= MEMCG_DATA_OBJEXTS;
2220#endif
2221retry:
2222 old_exts = READ_ONCE(slab->obj_exts);
2223 handle_failed_objexts_alloc(slab, old_exts, vec);
2224
2225 if (new_slab) {
2226 /*
2227 * If the slab is brand new and nobody can yet access its
2228 * obj_exts, no synchronization is required and obj_exts can
2229 * be simply assigned.
2230 */
2231 slab->obj_exts = new_exts;
2232 } else if (old_exts & ~OBJEXTS_FLAGS_MASK) {
2233 /*
2234 * If the slab is already in use, somebody can allocate and
2235 * assign slabobj_exts in parallel. In this case the existing
2236 * objcg vector should be reused.
2237 */
2238 if (unlikely(!allow_spin))
2239 kfree_nolock(vec);
2240 else
2241 kfree(vec);
2242 return 0;
2243 } else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) {
2244 /* Retry if a racing thread changed slab->obj_exts from under us. */
2245 goto retry;
2246 }
2247
2248 if (allow_spin)
2249 kmemleak_not_leak(vec);
2250 return 0;
2251}
2252
2253static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)
2254{
2255 struct slabobj_ext *obj_exts;
2256
2257 obj_exts = (struct slabobj_ext *)slab_obj_exts(slab);
2258 if (!obj_exts) {
2259 /*
2260 * If obj_exts allocation failed, slab->obj_exts is set to
2261 * OBJEXTS_ALLOC_FAIL. In this case, we end up here and should
2262 * clear the flag.
2263 */
2264 slab->obj_exts = 0;
2265 return;
2266 }
2267
2268 if (obj_exts_in_slab(slab->slab_cache, slab)) {
2269 slab->obj_exts = 0;
2270 return;
2271 }
2272
2273 if (allow_spin)
2274 kfree(obj_exts);
2275 else
2276 kfree_nolock(obj_exts);
2277 slab->obj_exts = 0;
2278}
2279
2280/*
2281 * Try to allocate slabobj_ext array from unused space.
2282 * This function must be called on a freshly allocated slab to prevent
2283 * concurrency problems.
2284 */
2285static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
2286{
2287 void *addr;
2288 unsigned long obj_exts;
2289
2290 if (!need_slab_obj_exts(s))
2291 return;
2292
2293 if (obj_exts_fit_within_slab_leftover(s, slab)) {
2294 addr = slab_address(slab) + obj_exts_offset_in_slab(s, slab);
2295 addr = kasan_reset_tag(addr);
2296 obj_exts = (unsigned long)addr;
2297
2298 get_slab_obj_exts(obj_exts);
2299 memset(addr, 0, obj_exts_size_in_slab(slab));
2300 put_slab_obj_exts(obj_exts);
2301
2302#ifdef CONFIG_MEMCG
2303 obj_exts |= MEMCG_DATA_OBJEXTS;
2304#endif
2305 slab->obj_exts = obj_exts;
2306 } else if (s->flags & SLAB_OBJ_EXT_IN_OBJ) {
2307 unsigned int offset = obj_exts_offset_in_object(s);
2308
2309 obj_exts = (unsigned long)slab_address(slab);
2310 obj_exts += s->red_left_pad;
2311 obj_exts += offset;
2312
2313 get_slab_obj_exts(obj_exts);
2314 for_each_object(addr, s, slab_address(slab), slab->objects)
2315 memset(kasan_reset_tag(addr) + offset, 0, slab_obj_ext_size(slab));
2316 put_slab_obj_exts(obj_exts);
2317
2318#ifdef CONFIG_MEMCG
2319 obj_exts |= MEMCG_DATA_OBJEXTS;
2320#endif
2321 slab->obj_exts = obj_exts;
2322 slab_set_obj_exts_in_object(slab);
2323 }
2324}
2325
2326#else /* CONFIG_SLAB_OBJ_EXT */
2327
2328static inline void mark_obj_codetag_empty(const void *obj)
2329{
2330}
2331
2332static inline void init_slab_obj_exts(struct slab *slab)
2333{
2334}
2335
2336static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
2337 gfp_t gfp, unsigned int alloc_flags)
2338{
2339 return 0;
2340}
2341
2342static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)
2343{
2344}
2345
2346static inline void alloc_slab_obj_exts_early(struct kmem_cache *s,
2347 struct slab *slab)
2348{
2349}
2350
2351#endif /* CONFIG_SLAB_OBJ_EXT */
2352
2353#ifdef CONFIG_MEM_ALLOC_PROFILING
2354
2355static inline unsigned long
2356prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
2357 gfp_t flags, unsigned int alloc_flags, void *p)
2358{
2359 if (!slab_obj_exts(slab)) {
2360 if (is_kfence_address(p))
2361 return 0;
2362
2363 if (alloc_slab_obj_exts(slab, s, flags, alloc_flags)) {
2364 pr_warn_once("%s, %s: Failed to create slab extension vector!\n",
2365 __func__, s->name);
2366 return 0;
2367 }
2368 }
2369
2370 return slab_obj_exts(slab);
2371}
2372
2373
2374/* Should be called only if mem_alloc_profiling_enabled() */
2375static noinline void
2376__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
2377 unsigned int alloc_flags)
2378{
2379 unsigned long obj_exts;
2380 struct slabobj_ext *obj_ext;
2381 struct slab *slab;
2382
2383 if (!object)
2384 return;
2385
2386 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
2387 return;
2388
2389 if (alloc_flags & SLAB_ALLOC_NO_RECURSE)
2390 return;
2391
2392 slab = virt_to_slab(object);
2393 obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object);
2394 /*
2395 * Currently obj_exts is used only for allocation profiling.
2396 * If other users appear then mem_alloc_profiling_enabled()
2397 * check should be added before alloc_tag_add().
2398 */
2399 if (obj_exts) {
2400 union codetag_ref *ref;
2401
2402 get_slab_obj_exts(obj_exts);
2403
2404 obj_ext = slab_obj_ext(s, slab, obj_exts, object);
2405 ref = slab_obj_ext_codetag_ref(slab, obj_ext);
2406
2407 alloc_tag_add(ref, current->alloc_tag, s->size);
2408
2409 put_slab_obj_exts(obj_exts);
2410 } else {
2411 /*
2412 * KFENCE allocations are rare and the amount of outstanding
2413 * ones is limited to a small number so it's not worth setting
2414 * tags as inaccurate because of them.
2415 */
2416 if (!is_kfence_address(object))
2417 alloc_tag_set_inaccurate(current->alloc_tag);
2418 }
2419}
2420
2421static inline void
2422alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
2423 unsigned int alloc_flags)
2424{
2425 if (mem_alloc_profiling_enabled())
2426 __alloc_tagging_slab_alloc_hook(s, object, flags, alloc_flags);
2427}
2428
2429/* Should be called only if mem_alloc_profiling_enabled() */
2430static noinline void
2431__alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2432 int objects)
2433{
2434 unsigned long obj_exts;
2435
2436 /* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */
2437 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
2438 return;
2439
2440 obj_exts = slab_obj_exts(slab);
2441 if (!obj_exts)
2442 return;
2443
2444 get_slab_obj_exts(obj_exts);
2445 for (int i = 0; i < objects; i++) {
2446 struct slabobj_ext *ext;
2447
2448 ext = slab_obj_ext(s, slab, obj_exts, p[i]);
2449 alloc_tag_sub(slab_obj_ext_codetag_ref(slab, ext), s->size);
2450 }
2451 put_slab_obj_exts(obj_exts);
2452}
2453
2454static inline void
2455alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2456 int objects)
2457{
2458 if (mem_alloc_profiling_enabled())
2459 __alloc_tagging_slab_free_hook(s, slab, p, objects);
2460}
2461
2462/*
2463 * Make sure the static key used by slab_obj_ext_has_codetag() reflects the
2464 * value of !mem_alloc_profiling_permanently_disabled()
2465 *
2466 * Any later mem alloc profiling shutdown won't be reflected in the static key
2467 * because obj_exts with codetags might already exist.
2468 */
2469static void __init slab_obj_ext_has_codetag_init(void)
2470{
2471 bool need_codetag = !mem_alloc_profiling_permanently_disabled();
2472
2473 if (need_codetag != static_key_enabled(&slab_obj_ext_has_codetag_key)) {
2474 if (need_codetag)
2475 static_branch_enable(&slab_obj_ext_has_codetag_key);
2476 else
2477 static_branch_disable(&slab_obj_ext_has_codetag_key);
2478 }
2479}
2480
2481#else /* CONFIG_MEM_ALLOC_PROFILING */
2482
2483static inline void
2484alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
2485 unsigned int alloc_flags)
2486{
2487}
2488
2489static inline void
2490alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2491 int objects)
2492{
2493}
2494
2495static inline void slab_obj_ext_has_codetag_init(void)
2496{
2497}
2498
2499#endif /* CONFIG_MEM_ALLOC_PROFILING */
2500
2501
2502#ifdef CONFIG_MEMCG
2503
2504static void memcg_alloc_abort_single(struct kmem_cache *s, void *object);
2505
2506static __fastpath_inline
2507bool memcg_slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
2508 size_t size, void **p,
2509 const struct slab_alloc_context *ac)
2510{
2511 if (likely(!memcg_kmem_online()))
2512 return true;
2513
2514 if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
2515 return true;
2516
2517 if (likely(__memcg_slab_post_alloc_hook(s, ac->lru, flags,
2518 ac->alloc_flags, size, p)))
2519 return true;
2520
2521 if (likely(size == 1)) {
2522 memcg_alloc_abort_single(s, *p);
2523 *p = NULL;
2524 } else {
2525 kmem_cache_free_bulk(s, size, p);
2526 }
2527
2528 return false;
2529}
2530
2531static __fastpath_inline
2532void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2533 int objects)
2534{
2535 unsigned long obj_exts;
2536
2537 if (!memcg_kmem_online())
2538 return;
2539
2540 obj_exts = slab_obj_exts(slab);
2541 if (likely(!obj_exts))
2542 return;
2543
2544 if (!slab_needs_objcg(slab))
2545 return;
2546
2547 get_slab_obj_exts(obj_exts);
2548 __memcg_slab_free_hook(s, slab, p, objects, obj_exts);
2549 put_slab_obj_exts(obj_exts);
2550}
2551
2552static __fastpath_inline
2553bool memcg_slab_post_charge(void *p, gfp_t flags)
2554{
2555 unsigned long obj_exts;
2556 struct slabobj_ext *obj_ext;
2557 struct kmem_cache *s;
2558 struct page *page;
2559 struct slab *slab;
2560
2561 page = virt_to_page(p);
2562 if (PageLargeKmalloc(page)) {
2563 unsigned int order;
2564 int size;
2565
2566 if (PageMemcgKmem(page))
2567 return true;
2568
2569 order = large_kmalloc_order(page);
2570 if (__memcg_kmem_charge_page(page, flags, order))
2571 return false;
2572
2573 /*
2574 * This page has already been accounted in the global stats but
2575 * not in the memcg stats. So, subtract from the global and use
2576 * the interface which adds to both global and memcg stats.
2577 */
2578 size = PAGE_SIZE << order;
2579 mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, -size);
2580 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, size);
2581 return true;
2582 }
2583
2584 slab = page_slab(page);
2585 s = slab->slab_cache;
2586
2587 /*
2588 * Ignore KMALLOC_NORMAL cache to avoid possible circular dependency
2589 * of slab_obj_exts being allocated from the same slab and thus the slab
2590 * becoming effectively unfreeable.
2591 */
2592 if (!cache_needs_objcg(s))
2593 return true;
2594
2595 /* Ignore already charged objects. */
2596 obj_exts = slab_obj_exts(slab);
2597 if (obj_exts) {
2598 get_slab_obj_exts(obj_exts);
2599 obj_ext = slab_obj_ext(s, slab, obj_exts, p);
2600 if (unlikely(slab_obj_ext_objcg(slab, obj_ext))) {
2601 put_slab_obj_exts(obj_exts);
2602 return true;
2603 }
2604 put_slab_obj_exts(obj_exts);
2605 }
2606
2607 return __memcg_slab_post_alloc_hook(s, NULL, flags, SLAB_ALLOC_DEFAULT,
2608 1, &p);
2609}
2610
2611#else /* CONFIG_MEMCG */
2612static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s,
2613 gfp_t flags,
2614 size_t size, void **p,
2615 const struct slab_alloc_context *ac)
2616{
2617 return true;
2618}
2619
2620static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
2621 void **p, int objects)
2622{
2623}
2624
2625static inline bool memcg_slab_post_charge(void *p, gfp_t flags)
2626{
2627 return true;
2628}
2629#endif /* CONFIG_MEMCG */
2630
2631#ifdef CONFIG_SLUB_RCU_DEBUG
2632static void slab_free_after_rcu_debug(struct rcu_head *rcu_head);
2633
2634struct rcu_delayed_free {
2635 struct rcu_head head;
2636 void *object;
2637};
2638#endif
2639
2640/*
2641 * Hooks for other subsystems that check memory allocations. In a typical
2642 * production configuration these hooks all should produce no code at all.
2643 *
2644 * Returns true if freeing of the object can proceed, false if its reuse
2645 * was delayed by CONFIG_SLUB_RCU_DEBUG or KASAN quarantine, or it was returned
2646 * to KFENCE.
2647 *
2648 * For objects allocated via kmalloc_nolock(), only a subset of alloc hooks
2649 * are invoked, so some free hooks must handle asymmetric hook calls.
2650 *
2651 * Alloc hooks called for kmalloc_nolock():
2652 * - kmsan_slab_alloc()
2653 * - kasan_slab_alloc()
2654 * - memcg_slab_post_alloc_hook()
2655 * - alloc_tagging_slab_alloc_hook()
2656 *
2657 * Free hooks that must handle missing corresponding alloc hooks:
2658 * - kmemleak_free_recursive()
2659 * - kfence_free()
2660 *
2661 * Free hooks that have no alloc hook counterpart, and thus safe to call:
2662 * - debug_check_no_locks_freed()
2663 * - debug_check_no_obj_freed()
2664 * - __kcsan_check_access()
2665 */
2666static __always_inline
2667bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
2668 bool after_rcu_delay)
2669{
2670 /* Are the object contents still accessible? */
2671 bool still_accessible = (s->flags & SLAB_TYPESAFE_BY_RCU) && !after_rcu_delay;
2672
2673 kmemleak_free_recursive(x, s->flags);
2674 kmsan_slab_free(s, x);
2675
2676 debug_check_no_locks_freed(x, s->object_size);
2677
2678 if (!(s->flags & SLAB_DEBUG_OBJECTS))
2679 debug_check_no_obj_freed(x, s->object_size);
2680
2681 /* Use KCSAN to help debug racy use-after-free. */
2682 if (!still_accessible)
2683 __kcsan_check_access(x, s->object_size,
2684 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
2685
2686 if (kfence_free(x))
2687 return false;
2688
2689 /*
2690 * Give KASAN a chance to notice an invalid free operation before we
2691 * modify the object.
2692 */
2693 if (kasan_slab_pre_free(s, x))
2694 return false;
2695
2696#ifdef CONFIG_SLUB_RCU_DEBUG
2697 if (still_accessible) {
2698 struct rcu_delayed_free *delayed_free;
2699
2700 delayed_free = kmalloc_obj(*delayed_free, GFP_NOWAIT);
2701 if (delayed_free) {
2702 /*
2703 * Let KASAN track our call stack as a "related work
2704 * creation", just like if the object had been freed
2705 * normally via kfree_rcu().
2706 * We have to do this manually because the rcu_head is
2707 * not located inside the object.
2708 */
2709 kasan_record_aux_stack(x);
2710
2711 delayed_free->object = x;
2712 call_rcu(&delayed_free->head, slab_free_after_rcu_debug);
2713 return false;
2714 }
2715 }
2716#endif /* CONFIG_SLUB_RCU_DEBUG */
2717
2718 /*
2719 * As memory initialization might be integrated into KASAN,
2720 * kasan_slab_free and initialization memset's must be
2721 * kept together to avoid discrepancies in behavior.
2722 *
2723 * The initialization memset's clear the object and the metadata,
2724 * but don't touch the SLAB redzone.
2725 *
2726 * The object's freepointer is also avoided if stored outside the
2727 * object.
2728 */
2729 if (unlikely(init)) {
2730 int rsize;
2731 unsigned int inuse, orig_size;
2732
2733 inuse = get_info_end(s);
2734 orig_size = get_orig_size(s, x);
2735 if (!kasan_has_integrated_init())
2736 memset(kasan_reset_tag(x), 0, orig_size);
2737 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
2738 memset((char *)kasan_reset_tag(x) + inuse, 0,
2739 s->size - inuse - rsize);
2740 /*
2741 * Restore orig_size, otherwise kmalloc redzone overwritten
2742 * would be reported
2743 */
2744 set_orig_size(s, x, orig_size);
2745
2746 }
2747 /* KASAN might put x into memory quarantine, delaying its reuse. */
2748 return !kasan_slab_free(s, x, init, still_accessible, false);
2749}
2750
2751static __fastpath_inline
2752bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail,
2753 int *cnt)
2754{
2755
2756 void *object;
2757 void *next = *head;
2758 void *old_tail = *tail;
2759 bool init;
2760
2761 if (is_kfence_address(next)) {
2762 slab_free_hook(s, next, false, false);
2763 return false;
2764 }
2765
2766 /* Head and tail of the reconstructed freelist */
2767 *head = NULL;
2768 *tail = NULL;
2769
2770 init = slab_want_init_on_free(s);
2771
2772 do {
2773 object = next;
2774 next = get_freepointer(s, object);
2775
2776 /* If object's reuse doesn't have to be delayed */
2777 if (likely(slab_free_hook(s, object, init, false))) {
2778 /* Move object to the new freelist */
2779 set_freepointer(s, object, *head);
2780 *head = object;
2781 if (!*tail)
2782 *tail = object;
2783 } else {
2784 /*
2785 * Adjust the reconstructed freelist depth
2786 * accordingly if object's reuse is delayed.
2787 */
2788 --(*cnt);
2789 }
2790 } while (object != old_tail);
2791
2792 return *head != NULL;
2793}
2794
2795static inline void *setup_object(struct kmem_cache *s, void *object)
2796{
2797 setup_object_debug(s, object);
2798 object = kasan_init_slab_obj(s, object);
2799 if (unlikely(s->ctor)) {
2800 kasan_unpoison_new_object(s, object);
2801 s->ctor(object);
2802 kasan_poison_new_object(s, object);
2803 }
2804 return object;
2805}
2806
2807static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
2808 unsigned int alloc_flags, unsigned int capacity)
2809{
2810 struct slab_sheaf *sheaf;
2811 size_t sheaf_size;
2812
2813 /*
2814 * Prevent recursion to the same cache, or a deep stack of kmallocs of
2815 * varying sizes (sheaf capacity might differ for each kmalloc size
2816 * bucket)
2817 */
2818 if (s->flags & SLAB_KMALLOC)
2819 alloc_flags |= SLAB_ALLOC_NO_RECURSE;
2820
2821 sheaf_size = struct_size(sheaf, objects, capacity);
2822 sheaf = kmalloc_flags(sheaf_size, gfp | __GFP_ZERO, alloc_flags, NUMA_NO_NODE);
2823
2824 if (unlikely(!sheaf))
2825 return NULL;
2826
2827 sheaf->cache = s;
2828
2829 stat(s, SHEAF_ALLOC);
2830
2831 return sheaf;
2832}
2833
2834static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
2835 gfp_t gfp, unsigned int alloc_flags)
2836{
2837 if (alloc_flags & SLAB_ALLOC_NO_RECURSE)
2838 return NULL;
2839
2840 gfp &= ~OBJCGS_CLEAR_MASK;
2841
2842 return __alloc_empty_sheaf(s, gfp, alloc_flags, s->sheaf_capacity);
2843}
2844
2845static void __free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf,
2846 unsigned int free_flags)
2847{
2848 /*
2849 * If the sheaf was created with SLAB_ALLOC_NO_RECURSE flag then its
2850 * corresponding extension is NULL and alloc_tag_sub() will throw a
2851 * warning, therefore replace NULL with CODETAG_EMPTY to indicate
2852 * that the extension for this sheaf is expected to be NULL.
2853 */
2854 if (s->flags & SLAB_KMALLOC)
2855 mark_obj_codetag_empty(sheaf);
2856
2857 VM_WARN_ON_ONCE(sheaf->size > 0);
2858
2859 if (unlikely(free_flags & SLAB_FREE_NOLOCK))
2860 kfree_nolock(sheaf);
2861 else
2862 kfree(sheaf);
2863
2864 stat(s, SHEAF_FREE);
2865}
2866
2867static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
2868{
2869 __free_empty_sheaf(s, sheaf, SLAB_FREE_DEFAULT);
2870}
2871
2872static unsigned int
2873refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
2874 unsigned int max);
2875
2876static int refill_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf,
2877 gfp_t gfp)
2878{
2879 int to_fill = s->sheaf_capacity - sheaf->size;
2880 int filled;
2881
2882 if (!to_fill)
2883 return 0;
2884
2885 filled = refill_objects(s, &sheaf->objects[sheaf->size], gfp, to_fill,
2886 to_fill);
2887
2888 sheaf->size += filled;
2889
2890 stat_add(s, SHEAF_REFILL, filled);
2891
2892 if (filled < to_fill)
2893 return -ENOMEM;
2894
2895 return 0;
2896}
2897
2898/*
2899 * Maximum number of objects freed during a single flush of main pcs sheaf.
2900 * Translates directly to an on-stack array size.
2901 */
2902#define PCS_BATCH_MAX 32U
2903
2904static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
2905
2906/*
2907 * Free all objects from the main sheaf. In order to perform
2908 * __kmem_cache_free_bulk() outside of cpu_sheaves->lock, work in batches where
2909 * object pointers are moved to a on-stack array under the lock. To bound the
2910 * stack usage, limit each batch to PCS_BATCH_MAX.
2911 *
2912 * Must be called with s->cpu_sheaves->lock locked, returns with the lock
2913 * unlocked.
2914 *
2915 * Returns how many objects are remaining to be flushed
2916 */
2917static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s)
2918{
2919 struct slub_percpu_sheaves *pcs;
2920 unsigned int batch, remaining;
2921 void *objects[PCS_BATCH_MAX];
2922 struct slab_sheaf *sheaf;
2923
2924 slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
2925
2926 pcs = this_cpu_ptr(s->cpu_sheaves);
2927 sheaf = pcs->main;
2928
2929 batch = min(PCS_BATCH_MAX, sheaf->size);
2930
2931 sheaf->size -= batch;
2932 memcpy(objects, sheaf->objects + sheaf->size, batch * sizeof(void *));
2933
2934 remaining = sheaf->size;
2935
2936 local_unlock(&s->cpu_sheaves->lock);
2937
2938 __kmem_cache_free_bulk(s, batch, &objects[0]);
2939
2940 stat_add(s, SHEAF_FLUSH, batch);
2941
2942 return remaining;
2943}
2944
2945static void sheaf_flush_main(struct kmem_cache *s)
2946{
2947 unsigned int remaining;
2948
2949 do {
2950 local_lock(&s->cpu_sheaves->lock);
2951
2952 remaining = __sheaf_flush_main_batch(s);
2953
2954 } while (remaining);
2955}
2956
2957/*
2958 * Returns true if the main sheaf was at least partially flushed.
2959 */
2960static bool sheaf_try_flush_main(struct kmem_cache *s)
2961{
2962 unsigned int remaining;
2963 bool ret = false;
2964
2965 do {
2966 if (!local_trylock(&s->cpu_sheaves->lock))
2967 return ret;
2968
2969 ret = true;
2970 remaining = __sheaf_flush_main_batch(s);
2971
2972 } while (remaining);
2973
2974 return ret;
2975}
2976
2977/*
2978 * Free all objects from a sheaf that's unused, i.e. not linked to any
2979 * cpu_sheaves, so we need no locking and batching. The locking is also not
2980 * necessary when flushing cpu's sheaves (both spare and main) during cpu
2981 * hotremove as the cpu is not executing anymore.
2982 */
2983static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf)
2984{
2985 if (!sheaf->size)
2986 return;
2987
2988 stat_add(s, SHEAF_FLUSH, sheaf->size);
2989
2990 __kmem_cache_free_bulk(s, sheaf->size, &sheaf->objects[0]);
2991
2992 sheaf->size = 0;
2993}
2994
2995static bool __rcu_free_sheaf_prepare(struct kmem_cache *s,
2996 struct slab_sheaf *sheaf)
2997{
2998 bool init = slab_want_init_on_free(s);
2999 void **p = &sheaf->objects[0];
3000 unsigned int i = 0;
3001 bool pfmemalloc = false;
3002
3003 while (i < sheaf->size) {
3004 struct slab *slab = virt_to_slab(p[i]);
3005
3006 memcg_slab_free_hook(s, slab, p + i, 1);
3007 alloc_tagging_slab_free_hook(s, slab, p + i, 1);
3008
3009 if (unlikely(!slab_free_hook(s, p[i], init, true))) {
3010 p[i] = p[--sheaf->size];
3011 continue;
3012 }
3013
3014 if (slab_test_pfmemalloc(slab))
3015 pfmemalloc = true;
3016
3017 i++;
3018 }
3019
3020 return pfmemalloc;
3021}
3022
3023static void rcu_free_sheaf_nobarn(struct rcu_head *head)
3024{
3025 struct slab_sheaf *sheaf;
3026 struct kmem_cache *s;
3027
3028 sheaf = container_of(head, struct slab_sheaf, rcu_head);
3029 s = sheaf->cache;
3030
3031 __rcu_free_sheaf_prepare(s, sheaf);
3032
3033 sheaf_flush_unused(s, sheaf);
3034
3035 free_empty_sheaf(s, sheaf);
3036}
3037
3038/*
3039 * Caller needs to make sure migration is disabled in order to fully flush
3040 * single cpu's sheaves
3041 *
3042 * must not be called from an irq
3043 *
3044 * flushing operations are rare so let's keep it simple and flush to slabs
3045 * directly, skipping the barn
3046 */
3047static void pcs_flush_all(struct kmem_cache *s)
3048{
3049 struct slub_percpu_sheaves *pcs;
3050 struct slab_sheaf *spare, *rcu_free;
3051
3052 local_lock(&s->cpu_sheaves->lock);
3053 pcs = this_cpu_ptr(s->cpu_sheaves);
3054
3055 spare = pcs->spare;
3056 pcs->spare = NULL;
3057
3058 rcu_free = pcs->rcu_free;
3059 pcs->rcu_free = NULL;
3060
3061 local_unlock(&s->cpu_sheaves->lock);
3062
3063 if (spare) {
3064 sheaf_flush_unused(s, spare);
3065 free_empty_sheaf(s, spare);
3066 }
3067
3068 if (rcu_free)
3069 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn);
3070
3071 sheaf_flush_main(s);
3072}
3073
3074static void __pcs_flush_all_cpu(struct kmem_cache *s, unsigned int cpu)
3075{
3076 struct slub_percpu_sheaves *pcs;
3077
3078 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
3079
3080 /* The cpu is not executing anymore so we don't need pcs->lock */
3081 sheaf_flush_unused(s, pcs->main);
3082 if (pcs->spare) {
3083 sheaf_flush_unused(s, pcs->spare);
3084 free_empty_sheaf(s, pcs->spare);
3085 pcs->spare = NULL;
3086 }
3087
3088 if (pcs->rcu_free) {
3089 call_rcu(&pcs->rcu_free->rcu_head, rcu_free_sheaf_nobarn);
3090 pcs->rcu_free = NULL;
3091 }
3092}
3093
3094static void pcs_destroy(struct kmem_cache *s)
3095{
3096 int cpu;
3097
3098 /*
3099 * We may be unwinding cache creation that failed before or during the
3100 * allocation of this.
3101 */
3102 if (!s->cpu_sheaves)
3103 return;
3104
3105 /* pcs->main can only point to the bootstrap sheaf, nothing to free */
3106 if (!cache_has_sheaves(s))
3107 goto free_pcs;
3108
3109 for_each_possible_cpu(cpu) {
3110 struct slub_percpu_sheaves *pcs;
3111
3112 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
3113
3114 /* This can happen when unwinding failed cache creation. */
3115 if (!pcs->main)
3116 continue;
3117
3118 /*
3119 * We have already passed __kmem_cache_shutdown() so everything
3120 * was flushed and there should be no objects allocated from
3121 * slabs, otherwise kmem_cache_destroy() would have aborted.
3122 * Therefore something would have to be really wrong if the
3123 * warnings here trigger, and we should rather leave objects and
3124 * sheaves to leak in that case.
3125 */
3126
3127 WARN_ON(pcs->spare);
3128 WARN_ON(pcs->rcu_free);
3129
3130 if (!WARN_ON(pcs->main->size)) {
3131 free_empty_sheaf(s, pcs->main);
3132 pcs->main = NULL;
3133 }
3134 }
3135
3136free_pcs:
3137 free_percpu(s->cpu_sheaves);
3138 s->cpu_sheaves = NULL;
3139}
3140
3141static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn,
3142 bool allow_spin)
3143{
3144 struct slab_sheaf *empty = NULL;
3145 unsigned long flags;
3146
3147 if (!data_race(barn->nr_empty))
3148 return NULL;
3149
3150 if (likely(allow_spin))
3151 spin_lock_irqsave(&barn->lock, flags);
3152 else if (!spin_trylock_irqsave(&barn->lock, flags))
3153 return NULL;
3154
3155 if (likely(barn->nr_empty)) {
3156 empty = list_first_entry(&barn->sheaves_empty,
3157 struct slab_sheaf, barn_list);
3158 list_del(&empty->barn_list);
3159 barn->nr_empty--;
3160 }
3161
3162 spin_unlock_irqrestore(&barn->lock, flags);
3163
3164 return empty;
3165}
3166
3167/*
3168 * The following two functions are used mainly in cases where we have to undo an
3169 * intended action due to a race or cpu migration. Thus they do not check the
3170 * empty or full sheaf limits for simplicity.
3171 */
3172
3173static void barn_put_empty_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf)
3174{
3175 unsigned long flags;
3176
3177 spin_lock_irqsave(&barn->lock, flags);
3178
3179 list_add(&sheaf->barn_list, &barn->sheaves_empty);
3180 barn->nr_empty++;
3181
3182 spin_unlock_irqrestore(&barn->lock, flags);
3183}
3184
3185static void barn_put_full_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf)
3186{
3187 unsigned long flags;
3188
3189 spin_lock_irqsave(&barn->lock, flags);
3190
3191 list_add(&sheaf->barn_list, &barn->sheaves_full);
3192 barn->nr_full++;
3193
3194 spin_unlock_irqrestore(&barn->lock, flags);
3195}
3196
3197static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn)
3198{
3199 struct slab_sheaf *sheaf = NULL;
3200 unsigned long flags;
3201
3202 if (!data_race(barn->nr_full) && !data_race(barn->nr_empty))
3203 return NULL;
3204
3205 spin_lock_irqsave(&barn->lock, flags);
3206
3207 if (barn->nr_full) {
3208 sheaf = list_first_entry(&barn->sheaves_full, struct slab_sheaf,
3209 barn_list);
3210 list_del(&sheaf->barn_list);
3211 barn->nr_full--;
3212 } else if (barn->nr_empty) {
3213 sheaf = list_first_entry(&barn->sheaves_empty,
3214 struct slab_sheaf, barn_list);
3215 list_del(&sheaf->barn_list);
3216 barn->nr_empty--;
3217 }
3218
3219 spin_unlock_irqrestore(&barn->lock, flags);
3220
3221 return sheaf;
3222}
3223
3224/*
3225 * If a full sheaf is available, return it and put the supplied empty one to
3226 * barn. We ignore the limit on empty sheaves as the number of sheaves doesn't
3227 * change.
3228 */
3229static struct slab_sheaf *
3230barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty,
3231 bool allow_spin)
3232{
3233 struct slab_sheaf *full = NULL;
3234 unsigned long flags;
3235
3236 if (!data_race(barn->nr_full))
3237 return NULL;
3238
3239 if (likely(allow_spin))
3240 spin_lock_irqsave(&barn->lock, flags);
3241 else if (!spin_trylock_irqsave(&barn->lock, flags))
3242 return NULL;
3243
3244 if (likely(barn->nr_full)) {
3245 full = list_first_entry(&barn->sheaves_full, struct slab_sheaf,
3246 barn_list);
3247 list_del(&full->barn_list);
3248 list_add(&empty->barn_list, &barn->sheaves_empty);
3249 barn->nr_full--;
3250 barn->nr_empty++;
3251 }
3252
3253 spin_unlock_irqrestore(&barn->lock, flags);
3254
3255 return full;
3256}
3257
3258/*
3259 * If an empty sheaf is available, return it and put the supplied full one to
3260 * barn. But if there are too many full sheaves, reject this with -E2BIG.
3261 */
3262static struct slab_sheaf *
3263barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full,
3264 bool allow_spin)
3265{
3266 struct slab_sheaf *empty;
3267 unsigned long flags;
3268
3269 /* we don't repeat this check under barn->lock as it's not critical */
3270 if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES)
3271 return ERR_PTR(-E2BIG);
3272 if (!data_race(barn->nr_empty))
3273 return ERR_PTR(-ENOMEM);
3274
3275 if (likely(allow_spin))
3276 spin_lock_irqsave(&barn->lock, flags);
3277 else if (!spin_trylock_irqsave(&barn->lock, flags))
3278 return ERR_PTR(-EBUSY);
3279
3280 if (likely(barn->nr_empty)) {
3281 empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf,
3282 barn_list);
3283 list_del(&empty->barn_list);
3284 list_add(&full->barn_list, &barn->sheaves_full);
3285 barn->nr_empty--;
3286 barn->nr_full++;
3287 } else {
3288 empty = ERR_PTR(-ENOMEM);
3289 }
3290
3291 spin_unlock_irqrestore(&barn->lock, flags);
3292
3293 return empty;
3294}
3295
3296static void barn_init(struct node_barn *barn)
3297{
3298 spin_lock_init(&barn->lock);
3299 INIT_LIST_HEAD(&barn->sheaves_full);
3300 INIT_LIST_HEAD(&barn->sheaves_empty);
3301 barn->nr_full = 0;
3302 barn->nr_empty = 0;
3303}
3304
3305static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)
3306{
3307 LIST_HEAD(empty_list);
3308 LIST_HEAD(full_list);
3309 struct slab_sheaf *sheaf, *sheaf2;
3310 unsigned long flags;
3311
3312 spin_lock_irqsave(&barn->lock, flags);
3313
3314 list_splice_init(&barn->sheaves_full, &full_list);
3315 barn->nr_full = 0;
3316 list_splice_init(&barn->sheaves_empty, &empty_list);
3317 barn->nr_empty = 0;
3318
3319 spin_unlock_irqrestore(&barn->lock, flags);
3320
3321 list_for_each_entry_safe(sheaf, sheaf2, &full_list, barn_list) {
3322 sheaf_flush_unused(s, sheaf);
3323 free_empty_sheaf(s, sheaf);
3324 }
3325
3326 list_for_each_entry_safe(sheaf, sheaf2, &empty_list, barn_list)
3327 free_empty_sheaf(s, sheaf);
3328}
3329
3330/*
3331 * Slab allocation and freeing
3332 */
3333static inline struct slab *alloc_slab_page(gfp_t flags, int node,
3334 struct kmem_cache_order_objects oo,
3335 bool allow_spin)
3336{
3337 struct page *page;
3338 struct slab *slab;
3339 unsigned int order = oo_order(oo);
3340
3341 if (unlikely(!allow_spin))
3342 page = alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */,
3343 node, order);
3344 else if (node == NUMA_NO_NODE)
3345 page = alloc_frozen_pages(flags, order);
3346 else
3347 page = __alloc_frozen_pages(flags, order, node, NULL,
3348 ALLOC_DEFAULT);
3349
3350 if (!page)
3351 return NULL;
3352
3353 __SetPageSlab(page);
3354 slab = page_slab(page);
3355 if (page_is_pfmemalloc(page))
3356 slab_set_pfmemalloc(slab);
3357
3358 return slab;
3359}
3360
3361#ifdef CONFIG_SLAB_FREELIST_RANDOM
3362/* Pre-initialize the random sequence cache */
3363static int init_cache_random_seq(struct kmem_cache *s)
3364{
3365 unsigned int count = oo_objects(s->oo);
3366 int err;
3367
3368 /* Bailout if already initialised */
3369 if (s->random_seq)
3370 return 0;
3371
3372 err = cache_random_seq_create(s, count, GFP_KERNEL);
3373 if (err) {
3374 pr_err("SLUB: Unable to initialize free list for %s\n",
3375 s->name);
3376 return err;
3377 }
3378
3379 /* Transform to an offset on the set of pages */
3380 if (s->random_seq) {
3381 unsigned int i;
3382
3383 for (i = 0; i < count; i++)
3384 s->random_seq[i] *= s->size;
3385 }
3386 return 0;
3387}
3388
3389/* Initialize each random sequence freelist per cache */
3390static void __init init_freelist_randomization(void)
3391{
3392 struct kmem_cache *s;
3393
3394 mutex_lock(&slab_mutex);
3395
3396 list_for_each_entry(s, &slab_caches, list)
3397 init_cache_random_seq(s);
3398
3399 mutex_unlock(&slab_mutex);
3400}
3401
3402static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state);
3403
3404#else
3405static inline int init_cache_random_seq(struct kmem_cache *s)
3406{
3407 return 0;
3408}
3409static inline void init_freelist_randomization(void) { }
3410#endif /* CONFIG_SLAB_FREELIST_RANDOM */
3411
3412static __always_inline void account_slab(struct slab *slab, int order,
3413 struct kmem_cache *s, gfp_t gfp,
3414 unsigned int alloc_flags)
3415{
3416 if (memcg_kmem_online() &&
3417 (s->flags & SLAB_ACCOUNT) &&
3418 !slab_obj_exts(slab))
3419 alloc_slab_obj_exts(slab, s, gfp,
3420 alloc_flags | SLAB_ALLOC_NEW_SLAB);
3421
3422 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
3423 PAGE_SIZE << order);
3424}
3425
3426static __always_inline void unaccount_slab(struct slab *slab, int order,
3427 struct kmem_cache *s, bool allow_spin)
3428{
3429 /*
3430 * The slab object extensions should now be freed regardless of
3431 * whether mem_alloc_profiling_enabled() or not because profiling
3432 * might have been disabled after slab->obj_exts got allocated.
3433 */
3434 free_slab_obj_exts(slab, allow_spin);
3435
3436 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
3437 -(PAGE_SIZE << order));
3438}
3439
3440/* Allocate and initialize a slab without building its freelist. */
3441static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,
3442 unsigned int alloc_flags, int node)
3443{
3444 bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
3445 struct slab *slab;
3446 struct kmem_cache_order_objects oo = s->oo;
3447 gfp_t alloc_gfp;
3448 void *start;
3449
3450 flags &= gfp_allowed_mask;
3451
3452 flags |= s->allocflags;
3453
3454 /*
3455 * Let the initial higher-order allocation fail under memory pressure
3456 * so we fall-back to the minimum order allocation.
3457 */
3458 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
3459 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
3460 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
3461
3462 slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin);
3463 if (unlikely(!slab)) {
3464 oo = s->min;
3465 alloc_gfp = flags;
3466 /*
3467 * Allocation may have failed due to fragmentation.
3468 * Try a lower order alloc if possible
3469 */
3470 slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin);
3471 if (unlikely(!slab))
3472 return NULL;
3473 stat(s, ORDER_FALLBACK);
3474 }
3475
3476 /* Initializes frozen, inuse, and any extra 64bit-only flags */
3477 slab->counters = 0;
3478
3479 slab->objects = oo_objects(oo);
3480
3481#ifdef CONFIG_64BIT
3482 if (cache_needs_objcg(s))
3483 slab->obj_exts_needs_objcg = 1;
3484#endif
3485 slab->slab_cache = s;
3486
3487 kasan_poison_slab(slab);
3488
3489 start = slab_address(slab);
3490
3491 setup_slab_debug(s, slab, start);
3492 init_slab_obj_exts(slab);
3493 /*
3494 * Poison the slab before initializing the slabobj_ext array
3495 * to prevent the array from being overwritten.
3496 */
3497 alloc_slab_obj_exts_early(s, slab);
3498 account_slab(slab, oo_order(oo), s, flags, alloc_flags);
3499
3500 return slab;
3501}
3502
3503static struct slab *new_slab(struct kmem_cache *s, gfp_t flags,
3504 unsigned int alloc_flags, int node)
3505{
3506 if (unlikely(flags & GFP_SLAB_BUG_MASK))
3507 flags = kmalloc_fix_flags(flags);
3508
3509 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
3510
3511 flags &= GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK;
3512
3513 return allocate_slab(s, flags, alloc_flags, node);
3514}
3515
3516static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)
3517{
3518 struct page *page = slab_page(slab);
3519 int order = compound_order(page);
3520 int pages = 1 << order;
3521
3522 __slab_clear_pfmemalloc(slab);
3523 page->mapping = NULL;
3524 __ClearPageSlab(page);
3525 mm_account_reclaimed_pages(pages);
3526 unaccount_slab(slab, order, s, allow_spin);
3527 if (allow_spin)
3528 free_frozen_pages(page, order);
3529 else
3530 free_frozen_pages_nolock(page, order);
3531}
3532
3533static void free_new_slab_nolock(struct kmem_cache *s, struct slab *slab)
3534{
3535 /*
3536 * Since it was just allocated, we can skip the actions in
3537 * discard_slab() and free_slab().
3538 */
3539 __free_slab(s, slab, false);
3540}
3541
3542static void rcu_free_slab(struct rcu_head *h)
3543{
3544 struct slab *slab = container_of(h, struct slab, rcu_head);
3545
3546 __free_slab(slab->slab_cache, slab, true);
3547}
3548
3549static void free_slab(struct kmem_cache *s, struct slab *slab)
3550{
3551 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
3552 void *p;
3553
3554 slab_pad_check(s, slab);
3555 for_each_object(p, s, slab_address(slab), slab->objects)
3556 check_object(s, slab, p, SLUB_RED_INACTIVE);
3557 }
3558
3559 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
3560 call_rcu(&slab->rcu_head, rcu_free_slab);
3561 else
3562 __free_slab(s, slab, true);
3563}
3564
3565static void discard_slab(struct kmem_cache *s, struct slab *slab)
3566{
3567 dec_slabs_node(s, slab_nid(slab), slab->objects);
3568 free_slab(s, slab);
3569}
3570
3571static inline bool slab_test_node_partial(const struct slab *slab)
3572{
3573 return test_bit(SL_partial, &slab->flags.f);
3574}
3575
3576static inline void slab_set_node_partial(struct slab *slab)
3577{
3578 set_bit(SL_partial, &slab->flags.f);
3579}
3580
3581static inline void slab_clear_node_partial(struct slab *slab)
3582{
3583 clear_bit(SL_partial, &slab->flags.f);
3584}
3585
3586/*
3587 * Management of partially allocated slabs.
3588 */
3589static inline void set_node_partial_state(struct kmem_cache_node *n,
3590 struct slab *slab)
3591{
3592 slab_set_node_partial(slab);
3593 n->nr_partial++;
3594}
3595
3596static inline void
3597__add_partial(struct kmem_cache_node *n, struct slab *slab, enum add_mode mode)
3598{
3599 if (mode == ADD_TO_TAIL)
3600 list_add_tail(&slab->slab_list, &n->partial);
3601 else
3602 list_add(&slab->slab_list, &n->partial);
3603 set_node_partial_state(n, slab);
3604}
3605
3606static inline void add_partial(struct kmem_cache_node *n,
3607 struct slab *slab, enum add_mode mode)
3608{
3609 slab_lockdep_assert_held(&n->list_lock);
3610 __add_partial(n, slab, mode);
3611}
3612
3613static inline void clear_node_partial_state(struct kmem_cache_node *n,
3614 struct slab *slab)
3615{
3616 slab_clear_node_partial(slab);
3617 n->nr_partial--;
3618}
3619
3620static inline void remove_partial(struct kmem_cache_node *n,
3621 struct slab *slab)
3622{
3623 slab_lockdep_assert_held(&n->list_lock);
3624 list_del(&slab->slab_list);
3625 clear_node_partial_state(n, slab);
3626}
3627
3628/*
3629 * Called only for kmem_cache_debug() caches instead of remove_partial(), with a
3630 * slab from the n->partial list. Remove only a single object from the slab, do
3631 * the alloc_debug_processing() checks and leave the slab on the list, or move
3632 * it to full list if it was the last free object.
3633 */
3634static void *alloc_single_from_partial(struct kmem_cache *s,
3635 struct kmem_cache_node *n, struct slab *slab, int orig_size)
3636{
3637 void *object;
3638
3639 slab_lockdep_assert_held(&n->list_lock);
3640
3641#ifdef CONFIG_SLUB_DEBUG
3642 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
3643 if (!validate_slab_ptr(slab)) {
3644 slab_err(s, slab, "Not a valid slab page");
3645 return NULL;
3646 }
3647 }
3648#endif
3649
3650 object = slab->freelist;
3651 slab->freelist = get_freepointer(s, object);
3652 slab->inuse++;
3653
3654 if (!alloc_debug_processing(s, slab, object, orig_size)) {
3655 remove_partial(n, slab);
3656 return NULL;
3657 }
3658
3659 if (slab->inuse == slab->objects) {
3660 remove_partial(n, slab);
3661 add_full(s, n, slab);
3662 }
3663
3664 return object;
3665}
3666
3667/* Return the next free object in allocation order. */
3668static inline void *next_slab_obj(struct kmem_cache *s,
3669 struct slab_obj_iter *iter)
3670{
3671#ifdef CONFIG_SLAB_FREELIST_RANDOM
3672 if (iter->random) {
3673 unsigned long idx;
3674
3675 /*
3676 * If the target page allocation failed, the number of objects on the
3677 * page might be smaller than the usual size defined by the cache.
3678 */
3679 do {
3680 idx = s->random_seq[iter->pos];
3681 iter->pos++;
3682 if (iter->pos >= iter->freelist_count)
3683 iter->pos = 0;
3684 } while (unlikely(idx >= iter->page_limit));
3685
3686 return setup_object(s, (char *)iter->start + idx);
3687 }
3688#endif
3689 return setup_object(s, (char *)iter->start + iter->pos++ * s->size);
3690}
3691
3692/* Build a freelist from the objects not yet allocated from a fresh slab. */
3693static inline void build_slab_freelist(struct kmem_cache *s, struct slab *slab,
3694 struct slab_obj_iter *iter)
3695{
3696 unsigned int nr = slab->objects - slab->inuse;
3697 unsigned int i;
3698 void *cur, *next;
3699
3700 if (!nr) {
3701 slab->freelist = NULL;
3702 return;
3703 }
3704
3705 cur = next_slab_obj(s, iter);
3706 slab->freelist = cur;
3707
3708 for (i = 1; i < nr; i++) {
3709 next = next_slab_obj(s, iter);
3710 set_freepointer(s, cur, next);
3711 cur = next;
3712 }
3713
3714 set_freepointer(s, cur, NULL);
3715}
3716
3717/* Initialize an iterator over free objects in allocation order. */
3718static inline void init_slab_obj_iter(struct kmem_cache *s, struct slab *slab,
3719 struct slab_obj_iter *iter,
3720 bool allow_spin)
3721{
3722 iter->pos = 0;
3723 iter->start = fixup_red_left(s, slab_address(slab));
3724
3725#ifdef CONFIG_SLAB_FREELIST_RANDOM
3726 iter->random = (slab->objects >= 2 && s->random_seq);
3727 if (!iter->random)
3728 return;
3729
3730 iter->freelist_count = oo_objects(s->oo);
3731 iter->page_limit = slab->objects * s->size;
3732
3733 if (allow_spin) {
3734 iter->pos = get_random_u32_below(iter->freelist_count);
3735 } else {
3736 struct rnd_state *state;
3737
3738 /*
3739 * An interrupt or NMI handler might interrupt and change
3740 * the state in the middle, but that's safe.
3741 */
3742 state = &get_cpu_var(slab_rnd_state);
3743 iter->pos = prandom_u32_state(state) % iter->freelist_count;
3744 put_cpu_var(slab_rnd_state);
3745 }
3746#endif
3747}
3748
3749/*
3750 * Called only for kmem_cache_debug() caches to allocate from a freshly
3751 * allocated slab. Allocate a single object instead of whole freelist
3752 * and put the slab to the partial (or full) list.
3753 */
3754static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
3755 const struct slab_alloc_context *ac)
3756{
3757 bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags);
3758 struct kmem_cache_node *n;
3759 struct slab_obj_iter iter;
3760 bool needs_add_partial;
3761 unsigned long flags;
3762 void *object;
3763
3764 init_slab_obj_iter(s, slab, &iter, allow_spin);
3765 object = next_slab_obj(s, &iter);
3766 slab->inuse = 1;
3767
3768 needs_add_partial = (slab->objects > 1);
3769 build_slab_freelist(s, slab, &iter);
3770
3771 /* alloc_debug_processing() always expects a valid freepointer */
3772 set_freepointer(s, object, slab->freelist);
3773
3774 if (!alloc_debug_processing(s, slab, object, ac->orig_size)) {
3775 /*
3776 * It's not really expected that this would fail on a
3777 * freshly allocated slab, but a concurrent memory
3778 * corruption in theory could cause that.
3779 * Leak memory of allocated slab.
3780 */
3781 return NULL;
3782 }
3783
3784 n = get_node(s, slab_nid(slab));
3785 if (allow_spin) {
3786 spin_lock_irqsave(&n->list_lock, flags);
3787 } else if (!spin_trylock_irqsave(&n->list_lock, flags)) {
3788 /*
3789 * Unlucky, discard newly allocated slab.
3790 * The slab is not fully free, but it's fine as
3791 * objects are not allocated to users.
3792 */
3793 free_new_slab_nolock(s, slab);
3794 return NULL;
3795 }
3796
3797 if (needs_add_partial)
3798 add_partial(n, slab, ADD_TO_HEAD);
3799 else
3800 add_full(s, n, slab);
3801
3802 /*
3803 * Debug caches require nr_slabs updates under n->list_lock so validation
3804 * cannot race with slab (de)allocations and observe inconsistent state.
3805 */
3806 inc_slabs_node(s, slab_nid(slab), slab->objects);
3807 spin_unlock_irqrestore(&n->list_lock, flags);
3808
3809 return object;
3810}
3811
3812static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
3813
3814static bool get_partial_node_bulk(struct kmem_cache *s,
3815 struct kmem_cache_node *n,
3816 struct partial_bulk_context *pc,
3817 bool allow_spin)
3818{
3819 struct slab *slab, *slab2;
3820 struct slab *first = NULL, *last = NULL;
3821 unsigned int total_free = 0;
3822 unsigned long flags;
3823
3824 /* Racy check to avoid taking the lock unnecessarily. */
3825 if (!n || data_race(!n->nr_partial))
3826 return false;
3827
3828 INIT_LIST_HEAD(&pc->slabs);
3829
3830 if (allow_spin)
3831 spin_lock_irqsave(&n->list_lock, flags);
3832 else if (!spin_trylock_irqsave(&n->list_lock, flags))
3833 return false;
3834
3835 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
3836 struct freelist_counters flc;
3837 unsigned int slab_free;
3838
3839 if (!pfmemalloc_match(slab, pc->flags)) {
3840 if (first) {
3841 list_bulk_move_tail(&pc->slabs,
3842 &first->slab_list,
3843 &last->slab_list);
3844 first = NULL;
3845 }
3846 continue;
3847 }
3848
3849 /*
3850 * determine the number of free objects in the slab racily
3851 *
3852 * slab_free is a lower bound due to possible subsequent
3853 * concurrent freeing, so the caller may get more objects than
3854 * requested and must handle that
3855 */
3856 flc.counters = data_race(READ_ONCE(slab->counters));
3857 slab_free = flc.objects - flc.inuse;
3858
3859 /* we have already min and this would get us over the max */
3860 if (total_free >= pc->min_objects
3861 && total_free + slab_free > pc->max_objects)
3862 break;
3863
3864 if (!first)
3865 first = slab;
3866 last = slab;
3867 clear_node_partial_state(n, slab);
3868
3869 total_free += slab_free;
3870 if (total_free >= pc->max_objects)
3871 break;
3872 }
3873
3874 if (first)
3875 list_bulk_move_tail(&pc->slabs, &first->slab_list,
3876 &last->slab_list);
3877
3878 spin_unlock_irqrestore(&n->list_lock, flags);
3879 return total_free > 0;
3880}
3881
3882/*
3883 * Try to allocate object from a partial slab on a specific node.
3884 */
3885static void *get_from_partial_node(struct kmem_cache *s,
3886 struct kmem_cache_node *n,
3887 gfp_t gfp_flags,
3888 const struct slab_alloc_context *ac)
3889{
3890 struct slab *slab, *slab2;
3891 unsigned long flags;
3892 void *object = NULL;
3893
3894 /*
3895 * Racy check. If we mistakenly see no partial slabs then we
3896 * just allocate an empty slab. If we mistakenly try to get a
3897 * partial slab and there is none available then get_from_partial()
3898 * will return NULL.
3899 */
3900 if (!n || !n->nr_partial)
3901 return NULL;
3902
3903 if (alloc_flags_allow_spinning(ac->alloc_flags))
3904 spin_lock_irqsave(&n->list_lock, flags);
3905 else if (!spin_trylock_irqsave(&n->list_lock, flags))
3906 return NULL;
3907 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
3908
3909 struct freelist_counters old, new;
3910
3911 if (!pfmemalloc_match(slab, gfp_flags))
3912 continue;
3913
3914 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
3915 object = alloc_single_from_partial(s, n, slab,
3916 ac->orig_size);
3917 if (object)
3918 break;
3919 continue;
3920 }
3921
3922 /*
3923 * get a single object from the slab. This might race against
3924 * __slab_free(), which however has to take the list_lock if
3925 * it's about to make the slab fully free.
3926 */
3927 do {
3928 old.freelist = slab->freelist;
3929 old.counters = slab->counters;
3930
3931 new.freelist = get_freepointer(s, old.freelist);
3932 new.counters = old.counters;
3933 new.inuse++;
3934
3935 } while (!__slab_update_freelist(s, slab, &old, &new, "get_from_partial_node"));
3936
3937 object = old.freelist;
3938 if (!new.freelist)
3939 remove_partial(n, slab);
3940
3941 break;
3942 }
3943 spin_unlock_irqrestore(&n->list_lock, flags);
3944 return object;
3945}
3946
3947/*
3948 * Get an object from somewhere. Search in increasing NUMA distances.
3949 */
3950static void *get_from_any_partial(struct kmem_cache *s, gfp_t gfp_flags,
3951 const struct slab_alloc_context *ac)
3952{
3953#ifdef CONFIG_NUMA
3954 struct zonelist *zonelist;
3955 struct zoneref *z;
3956 struct zone *zone;
3957 enum zone_type highest_zoneidx = gfp_zone(gfp_flags);
3958 unsigned int cpuset_mems_cookie;
3959 bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags);
3960
3961 /*
3962 * The defrag ratio allows a configuration of the tradeoffs between
3963 * inter node defragmentation and node local allocations. A lower
3964 * defrag_ratio increases the tendency to do local allocations
3965 * instead of attempting to obtain partial slabs from other nodes.
3966 *
3967 * If the defrag_ratio is set to 0 then kmalloc() always
3968 * returns node local objects. If the ratio is higher then kmalloc()
3969 * may return off node objects because partial slabs are obtained
3970 * from other nodes and filled up.
3971 *
3972 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
3973 * (which makes defrag_ratio = 1000) then every (well almost)
3974 * allocation will first attempt to defrag slab caches on other nodes.
3975 * This means scanning over all nodes to look for partial slabs which
3976 * may be expensive if we do it every time we are trying to find a slab
3977 * with available objects.
3978 */
3979 if (!s->remote_node_defrag_ratio ||
3980 get_cycles() % 1024 > s->remote_node_defrag_ratio)
3981 return NULL;
3982
3983 do {
3984 /*
3985 * read_mems_allowed_begin() accesses current->mems_allowed_seq,
3986 * a seqcount_spinlock_t that is not NMI-safe. Do not access
3987 * current->mems_allowed_seq and avoid retry when GFP flags
3988 * indicate spinning is not allowed.
3989 */
3990 if (allow_spin)
3991 cpuset_mems_cookie = read_mems_allowed_begin();
3992
3993 zonelist = node_zonelist(mempolicy_slab_node(), gfp_flags);
3994 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
3995 struct kmem_cache_node *n;
3996
3997 n = get_node(s, zone_to_nid(zone));
3998
3999 if (n && cpuset_zone_allowed(zone, gfp_flags) &&
4000 n->nr_partial > s->min_partial) {
4001
4002 void *object = get_from_partial_node(s, n,
4003 gfp_flags, ac);
4004
4005 if (object) {
4006 /*
4007 * Don't check read_mems_allowed_retry()
4008 * here - if mems_allowed was updated in
4009 * parallel, that was a harmless race
4010 * between allocation and the cpuset
4011 * update
4012 */
4013 return object;
4014 }
4015 }
4016 }
4017 } while (allow_spin && read_mems_allowed_retry(cpuset_mems_cookie));
4018#endif /* CONFIG_NUMA */
4019 return NULL;
4020}
4021
4022/*
4023 * Get an object from a partial slab
4024 */
4025static void *get_from_partial(struct kmem_cache *s, int node, gfp_t flags,
4026 const struct slab_alloc_context *ac)
4027{
4028 int searchnode = node;
4029 void *object;
4030
4031 if (node == NUMA_NO_NODE)
4032 searchnode = numa_mem_id();
4033
4034 object = get_from_partial_node(s, get_node(s, searchnode), flags, ac);
4035 if (object || (node != NUMA_NO_NODE && (flags & __GFP_THISNODE)))
4036 return object;
4037
4038 return get_from_any_partial(s, flags, ac);
4039}
4040
4041static bool has_pcs_used(int cpu, struct kmem_cache *s)
4042{
4043 struct slub_percpu_sheaves *pcs;
4044
4045 if (!cache_has_sheaves(s))
4046 return false;
4047
4048 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
4049
4050 return (pcs->spare || pcs->rcu_free || pcs->main->size);
4051}
4052
4053/*
4054 * Flush percpu sheaves
4055 *
4056 * Called from CPU work handler with migration disabled.
4057 */
4058static void flush_cpu_sheaves(struct work_struct *w)
4059{
4060 struct kmem_cache *s;
4061 struct slub_flush_work *sfw;
4062
4063 sfw = container_of(w, struct slub_flush_work, work);
4064
4065 s = sfw->s;
4066
4067 if (cache_has_sheaves(s))
4068 pcs_flush_all(s);
4069}
4070
4071static void flush_all_cpus_locked(struct kmem_cache *s)
4072{
4073 struct slub_flush_work *sfw;
4074 unsigned int cpu;
4075
4076 lockdep_assert_cpus_held();
4077 mutex_lock(&flush_lock);
4078
4079 for_each_online_cpu(cpu) {
4080 sfw = &per_cpu(slub_flush, cpu);
4081 if (!has_pcs_used(cpu, s)) {
4082 sfw->skip = true;
4083 continue;
4084 }
4085 INIT_WORK(&sfw->work, flush_cpu_sheaves);
4086 sfw->skip = false;
4087 sfw->s = s;
4088 queue_work_on(cpu, flushwq, &sfw->work);
4089 }
4090
4091 for_each_online_cpu(cpu) {
4092 sfw = &per_cpu(slub_flush, cpu);
4093 if (sfw->skip)
4094 continue;
4095 flush_work(&sfw->work);
4096 }
4097
4098 mutex_unlock(&flush_lock);
4099}
4100
4101static void flush_all(struct kmem_cache *s)
4102{
4103 cpus_read_lock();
4104 flush_all_cpus_locked(s);
4105 cpus_read_unlock();
4106}
4107
4108struct deferred_percpu_work {
4109 struct llist_head objects;
4110 struct llist_head objects_by_rcu;
4111 struct llist_head rcu_sheaves;
4112 struct irq_work work;
4113};
4114
4115static void deferred_percpu_work_fn(struct irq_work *work);
4116
4117static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = {
4118 .objects = LLIST_HEAD_INIT(objects),
4119 .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu),
4120 .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves),
4121 .work = IRQ_WORK_INIT(deferred_percpu_work_fn),
4122};
4123
4124static void flush_rcu_sheaf(struct work_struct *w)
4125{
4126 struct slub_percpu_sheaves *pcs;
4127 struct slab_sheaf *rcu_free;
4128 struct slub_flush_work *sfw;
4129 struct kmem_cache *s;
4130
4131 sfw = container_of(w, struct slub_flush_work, work);
4132 s = sfw->s;
4133
4134 local_lock(&s->cpu_sheaves->lock);
4135 pcs = this_cpu_ptr(s->cpu_sheaves);
4136
4137 rcu_free = pcs->rcu_free;
4138 pcs->rcu_free = NULL;
4139
4140 local_unlock(&s->cpu_sheaves->lock);
4141
4142 if (rcu_free)
4143 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn);
4144}
4145
4146
4147/* needed for kvfree_rcu_barrier() */
4148void flush_rcu_sheaves_on_cache(struct kmem_cache *s)
4149{
4150 struct slub_flush_work *sfw;
4151 unsigned int cpu;
4152
4153 lockdep_assert_cpus_held();
4154 mutex_lock(&flush_lock);
4155
4156 for_each_online_cpu(cpu) {
4157 sfw = &per_cpu(slub_flush, cpu);
4158
4159 /*
4160 * we don't check if rcu_free sheaf exists - racing
4161 * __kfree_rcu_sheaf() might have just removed it.
4162 * by executing flush_rcu_sheaf() on the cpu we make
4163 * sure the __kfree_rcu_sheaf() finished its call_rcu()
4164 */
4165
4166 INIT_WORK(&sfw->work, flush_rcu_sheaf);
4167 sfw->s = s;
4168 queue_work_on(cpu, flushwq, &sfw->work);
4169 }
4170
4171 for_each_online_cpu(cpu) {
4172 sfw = &per_cpu(slub_flush, cpu);
4173 flush_work(&sfw->work);
4174 }
4175
4176 mutex_unlock(&flush_lock);
4177}
4178
4179void flush_all_rcu_sheaves(void)
4180{
4181 struct kmem_cache *s;
4182
4183 deferred_work_barrier();
4184
4185 cpus_read_lock();
4186 mutex_lock(&slab_mutex);
4187
4188 list_for_each_entry(s, &slab_caches, list) {
4189 if (!cache_has_sheaves(s))
4190 continue;
4191 flush_rcu_sheaves_on_cache(s);
4192 }
4193
4194 mutex_unlock(&slab_mutex);
4195 cpus_read_unlock();
4196
4197 rcu_barrier();
4198}
4199
4200static int slub_cpu_setup(unsigned int cpu)
4201{
4202 int nid = cpu_to_node(cpu);
4203 struct kmem_cache *s;
4204 int ret = 0;
4205
4206 /*
4207 * we never clear a nid so it's safe to do a quick check before taking
4208 * the mutex, and then recheck to handle parallel cpu hotplug safely
4209 */
4210 if (node_isset(nid, slab_barn_nodes))
4211 return 0;
4212
4213 mutex_lock(&slab_mutex);
4214
4215 if (node_isset(nid, slab_barn_nodes))
4216 goto out;
4217
4218 list_for_each_entry(s, &slab_caches, list) {
4219 struct node_barn *barn;
4220
4221 /*
4222 * barn might already exist if a previous callback failed midway
4223 */
4224 if (!cache_has_sheaves(s) || get_barn_node(s, nid))
4225 continue;
4226
4227 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, nid);
4228
4229 if (!barn) {
4230 ret = -ENOMEM;
4231 goto out;
4232 }
4233
4234 barn_init(barn);
4235 s->per_node[nid].barn = barn;
4236 }
4237 node_set(nid, slab_barn_nodes);
4238
4239out:
4240 mutex_unlock(&slab_mutex);
4241
4242 return ret;
4243}
4244
4245/*
4246 * Use the cpu notifier to insure that the cpu slabs are flushed when
4247 * necessary.
4248 */
4249static int slub_cpu_dead(unsigned int cpu)
4250{
4251 struct kmem_cache *s;
4252
4253 mutex_lock(&slab_mutex);
4254 list_for_each_entry(s, &slab_caches, list) {
4255 if (cache_has_sheaves(s))
4256 __pcs_flush_all_cpu(s, cpu);
4257 }
4258 mutex_unlock(&slab_mutex);
4259 return 0;
4260}
4261
4262#ifdef CONFIG_SLUB_DEBUG
4263static int count_free(struct slab *slab)
4264{
4265 return slab->objects - slab->inuse;
4266}
4267
4268static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
4269{
4270 return atomic_long_read(&n->total_objects);
4271}
4272
4273/* Supports checking bulk free of a constructed freelist */
4274static inline bool free_debug_processing(struct kmem_cache *s,
4275 struct slab *slab, void *head, void *tail, int *bulk_cnt,
4276 unsigned long addr, depot_stack_handle_t handle)
4277{
4278 bool checks_ok = false;
4279 void *object = head;
4280 int cnt = 0;
4281
4282 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
4283 if (!check_slab(s, slab))
4284 goto out;
4285 }
4286
4287 if (slab->inuse < *bulk_cnt) {
4288 slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",
4289 slab->inuse, *bulk_cnt);
4290 goto out;
4291 }
4292
4293next_object:
4294
4295 if (++cnt > *bulk_cnt)
4296 goto out_cnt;
4297
4298 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
4299 if (!free_consistency_checks(s, slab, object, addr))
4300 goto out;
4301 }
4302
4303 if (s->flags & SLAB_STORE_USER)
4304 set_track_update(s, object, TRACK_FREE, addr, handle);
4305 trace(s, slab, object, 0);
4306 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
4307 init_object(s, object, SLUB_RED_INACTIVE);
4308
4309 /* Reached end of constructed freelist yet? */
4310 if (object != tail) {
4311 object = get_freepointer(s, object);
4312 goto next_object;
4313 }
4314 checks_ok = true;
4315
4316out_cnt:
4317 if (cnt != *bulk_cnt) {
4318 slab_err(s, slab, "Bulk free expected %d objects but found %d\n",
4319 *bulk_cnt, cnt);
4320 *bulk_cnt = cnt;
4321 }
4322
4323out:
4324
4325 if (!checks_ok)
4326 slab_fix(s, "Object at 0x%p not freed", object);
4327
4328 return checks_ok;
4329}
4330#endif /* CONFIG_SLUB_DEBUG */
4331
4332#if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS)
4333static unsigned long count_partial(struct kmem_cache_node *n,
4334 int (*get_count)(struct slab *))
4335{
4336 unsigned long flags;
4337 unsigned long x = 0;
4338 struct slab *slab;
4339
4340 spin_lock_irqsave(&n->list_lock, flags);
4341 list_for_each_entry(slab, &n->partial, slab_list)
4342 x += get_count(slab);
4343 spin_unlock_irqrestore(&n->list_lock, flags);
4344 return x;
4345}
4346#endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */
4347
4348#ifdef CONFIG_SLUB_DEBUG
4349#define MAX_PARTIAL_TO_SCAN 10000
4350
4351static unsigned long count_partial_free_approx(struct kmem_cache_node *n)
4352{
4353 unsigned long flags;
4354 unsigned long x = 0;
4355 struct slab *slab;
4356
4357 spin_lock_irqsave(&n->list_lock, flags);
4358 if (n->nr_partial <= MAX_PARTIAL_TO_SCAN) {
4359 list_for_each_entry(slab, &n->partial, slab_list)
4360 x += slab->objects - slab->inuse;
4361 } else {
4362 /*
4363 * For a long list, approximate the total count of objects in
4364 * it to meet the limit on the number of slabs to scan.
4365 * Scan from both the list's head and tail for better accuracy.
4366 */
4367 unsigned long scanned = 0;
4368
4369 list_for_each_entry(slab, &n->partial, slab_list) {
4370 x += slab->objects - slab->inuse;
4371 if (++scanned == MAX_PARTIAL_TO_SCAN / 2)
4372 break;
4373 }
4374 list_for_each_entry_reverse(slab, &n->partial, slab_list) {
4375 x += slab->objects - slab->inuse;
4376 if (++scanned == MAX_PARTIAL_TO_SCAN)
4377 break;
4378 }
4379 x = mult_frac(x, n->nr_partial, scanned);
4380 x = min(x, node_nr_objs(n));
4381 }
4382 spin_unlock_irqrestore(&n->list_lock, flags);
4383 return x;
4384}
4385
4386static noinline void
4387slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
4388{
4389 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
4390 DEFAULT_RATELIMIT_BURST);
4391 int cpu = raw_smp_processor_id();
4392 int node;
4393 struct kmem_cache_node *n;
4394
4395 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
4396 return;
4397
4398 pr_warn("SLUB: Unable to allocate memory on CPU %u (of node %d) on node %d, gfp=%#x(%pGg)\n",
4399 cpu, cpu_to_node(cpu), nid, gfpflags, &gfpflags);
4400 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
4401 s->name, s->object_size, s->size, oo_order(s->oo),
4402 oo_order(s->min));
4403
4404 if (oo_order(s->min) > get_order(s->object_size))
4405 pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n",
4406 s->name);
4407
4408 for_each_kmem_cache_node(s, node, n) {
4409 unsigned long nr_slabs;
4410 unsigned long nr_objs;
4411 unsigned long nr_free;
4412
4413 nr_free = count_partial_free_approx(n);
4414 nr_slabs = node_nr_slabs(n);
4415 nr_objs = node_nr_objs(n);
4416
4417 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
4418 node, nr_slabs, nr_objs, nr_free);
4419 }
4420}
4421#else /* CONFIG_SLUB_DEBUG */
4422static inline void
4423slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { }
4424#endif
4425
4426static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
4427{
4428 if (unlikely(slab_test_pfmemalloc(slab)))
4429 return gfp_pfmemalloc_allowed(gfpflags);
4430
4431 return true;
4432}
4433
4434/*
4435 * Get the slab's freelist and do not freeze it.
4436 *
4437 * Assumes the slab is isolated from node partial list and not frozen.
4438 *
4439 * Assumes this is performed only for caches without debugging so we
4440 * don't need to worry about adding the slab to the full list.
4441 */
4442static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *slab,
4443 unsigned int *count)
4444{
4445 struct freelist_counters old, new;
4446
4447 do {
4448 old.freelist = slab->freelist;
4449 old.counters = slab->counters;
4450
4451 new.freelist = NULL;
4452 new.counters = old.counters;
4453 VM_WARN_ON_ONCE(new.frozen);
4454
4455 new.inuse = old.objects;
4456
4457 } while (!slab_update_freelist(s, slab, &old, &new, "get_freelist_nofreeze"));
4458
4459 *count = old.objects - old.inuse;
4460 return old.freelist;
4461}
4462
4463/*
4464 * If the object has been wiped upon free, make sure it's fully initialized by
4465 * zeroing out freelist pointer.
4466 *
4467 * Note that we also wipe custom freelist pointers.
4468 */
4469static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
4470 void *obj)
4471{
4472 if (unlikely(slab_want_init_on_free(s)) && obj &&
4473 !freeptr_outside_object(s))
4474 memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
4475 0, sizeof(void *));
4476}
4477
4478static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab,
4479 void **p, unsigned int count, bool allow_spin)
4480{
4481 unsigned int allocated = 0;
4482 struct slab_obj_iter iter;
4483 bool needs_add_partial = true;
4484 unsigned long flags;
4485
4486 /*
4487 * Are we going to put the slab on the partial list?
4488 * Note slab->inuse is 0 on a new slab.
4489 */
4490 if (count >= slab->objects) {
4491 needs_add_partial = false;
4492 count = slab->objects;
4493 }
4494
4495 init_slab_obj_iter(s, slab, &iter, allow_spin);
4496
4497 while (allocated < count) {
4498 p[allocated] = next_slab_obj(s, &iter);
4499 allocated++;
4500 }
4501 slab->inuse = count;
4502 build_slab_freelist(s, slab, &iter);
4503
4504 if (needs_add_partial) {
4505 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
4506
4507 if (allow_spin) {
4508 spin_lock_irqsave(&n->list_lock, flags);
4509 } else if (!spin_trylock_irqsave(&n->list_lock, flags)) {
4510 /*
4511 * Unlucky, discard newly allocated slab.
4512 * The slab is not fully free, but it's fine as
4513 * objects are not allocated to users.
4514 */
4515 free_new_slab_nolock(s, slab);
4516 return 0;
4517 }
4518 add_partial(n, slab, ADD_TO_HEAD);
4519 spin_unlock_irqrestore(&n->list_lock, flags);
4520 }
4521
4522 inc_slabs_node(s, slab_nid(slab), slab->objects);
4523 return allocated;
4524}
4525
4526/*
4527 * Slow path. We failed to allocate via percpu sheaves or they are not available
4528 * due to bootstrap or debugging enabled or SLUB_TINY.
4529 *
4530 * We try to allocate from partial slab lists and fall back to allocating a new
4531 * slab.
4532 */
4533static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
4534 const struct slab_alloc_context *ac)
4535{
4536 bool allow_spin = alloc_flags_allow_spinning(ac->alloc_flags);
4537 gfp_t trynode_flags;
4538 void *object;
4539 struct slab *slab;
4540 bool try_thisnode = true;
4541
4542 stat(s, ALLOC_SLOWPATH);
4543
4544new_objects:
4545
4546 trynode_flags = gfpflags;
4547 /*
4548 * When a preferred node is indicated but no __GFP_THISNODE
4549 *
4550 * 1) try to get a partial slab from target node only by having
4551 * __GFP_THISNODE in trynode_flags for get_from_partial()
4552 * 2) if 1) failed, try to allocate a new slab from target node with
4553 * (at most) GFP_NOWAIT | __GFP_THISNODE opportunistically
4554 * 3) if 2) failed, retry with original gfpflags which will allow
4555 * get_from_partial() try partial lists of other nodes before
4556 * potentially allocating new page from other nodes
4557 */
4558 if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
4559 && try_thisnode)) {
4560 trynode_flags &= GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_ACCOUNT;
4561 trynode_flags |= __GFP_NOWARN | __GFP_THISNODE;
4562 }
4563
4564 object = get_from_partial(s, node, trynode_flags, ac);
4565 if (object)
4566 goto success;
4567
4568 slab = new_slab(s, trynode_flags, ac->alloc_flags, node);
4569
4570 if (unlikely(!slab)) {
4571 if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
4572 && try_thisnode) {
4573 try_thisnode = false;
4574 goto new_objects;
4575 }
4576 slab_out_of_memory(s, gfpflags, node);
4577 return NULL;
4578 }
4579
4580 stat(s, ALLOC_SLAB);
4581
4582 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
4583 object = alloc_single_from_new_slab(s, slab, ac);
4584
4585 if (likely(object))
4586 goto success;
4587 } else {
4588 /* we don't need to check SLAB_STORE_USER here */
4589 if (alloc_from_new_slab(s, slab, &object, 1, allow_spin))
4590 return object;
4591 }
4592
4593 if (allow_spin)
4594 goto new_objects;
4595
4596 /* This could cause an endless loop. Fail instead. */
4597 return NULL;
4598
4599success:
4600 if (kmem_cache_debug_flags(s, SLAB_STORE_USER))
4601 set_track(s, object, TRACK_ALLOC, ac->caller_addr, gfpflags);
4602
4603 return object;
4604}
4605
4606static __always_inline int apply_strict_numa_policy(int node)
4607{
4608#ifdef CONFIG_NUMA
4609 if (static_branch_unlikely(&strict_numa) &&
4610 node == NUMA_NO_NODE) {
4611
4612 struct mempolicy *mpol = current->mempolicy;
4613
4614 if (mpol) {
4615 /*
4616 * Special BIND rule support. If the local node
4617 * is in permitted set then do not redirect
4618 * to a particular node.
4619 * Otherwise we apply the memory policy to get
4620 * the node we need to allocate on.
4621 */
4622 if (mpol->mode != MPOL_BIND ||
4623 !node_isset(numa_mem_id(), mpol->nodes))
4624 node = mempolicy_slab_node();
4625 }
4626 }
4627#endif
4628 return node;
4629}
4630
4631static __fastpath_inline
4632struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
4633{
4634 flags &= gfp_allowed_mask;
4635
4636 might_alloc(flags);
4637
4638 if (unlikely(should_failslab(s, flags)))
4639 return NULL;
4640
4641 return s;
4642}
4643
4644static __fastpath_inline
4645bool slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, size_t size,
4646 void **p, const struct slab_alloc_context *ac)
4647{
4648 bool init = slab_want_init_on_alloc(flags, s);
4649 unsigned int zero_size = s->object_size;
4650 gfp_t init_flags = flags & gfp_allowed_mask;
4651 bool kasan_init = false;
4652
4653 /*
4654 * For kmalloc object, the allocated size (object_size) can be larger
4655 * than the requested size (orig_size). We however need to zero the
4656 * whole object_size to handle possible later krealloc() with
4657 *__GFP_ZERO properly.
4658 *
4659 * But if we keep track of the requested size, krealloc() uses that
4660 * information. Additionally if red zoning is enabled, the extra space
4661 * is also red zone, so we should not overwrite it. So limit zeroing to
4662 * orig_size if we track it.
4663 */
4664 if (slub_debug_orig_size(s))
4665 zero_size = ac->orig_size;
4666
4667 /*
4668 * ARM64 can set memory tags and zero the memory using a single
4669 * instruction. Since HW_TAGS KASAN uses that while tagging the object,
4670 * separate zeroing is unnecessary.
4671 *
4672 * However, KASAN never zeroes memory when slab_debug is enabled to
4673 * avoid overwriting SLUB redzones. This does not lead to a performance
4674 * penalty on production builds, as slab_debug is not intended to be
4675 * enabled there.
4676 */
4677 if (kasan_has_integrated_init() && !__slub_debug_enabled()) {
4678 kasan_init = init;
4679 init = false;
4680 }
4681
4682 for (size_t i = 0; i < size; i++) {
4683 p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init);
4684
4685 /*
4686 * memset and hooks come after KASAN as p[i] might get tagged
4687 *
4688 * kfence zeroes the object instead of SLUB to avoid overwriting
4689 * its own redzone starting at orig_size, which could happen
4690 * with SLUB zeroing full s->object_size
4691 */
4692 if (init && p[i] && !is_kfence_address(p[i]))
4693 memset(p[i], 0, zero_size);
4694
4695 if (alloc_flags_allow_spinning(ac->alloc_flags))
4696 kmemleak_alloc_recursive(p[i], s->object_size, 1,
4697 s->flags, init_flags);
4698 kmsan_slab_alloc(s, p[i], init_flags);
4699 alloc_tagging_slab_alloc_hook(s, p[i], flags, ac->alloc_flags);
4700 }
4701
4702 return memcg_slab_post_alloc_hook(s, flags, size, p, ac);
4703}
4704
4705/*
4706 * Replace the empty main sheaf with a (at least partially) full sheaf.
4707 *
4708 * Must be called with the cpu_sheaves local lock locked. If successful, returns
4709 * the pcs pointer and the local lock locked (possibly on a different cpu than
4710 * initially called). If not successful, returns NULL and the local lock
4711 * unlocked.
4712 */
4713static struct slub_percpu_sheaves *
4714__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
4715 gfp_t gfp, unsigned int alloc_flags)
4716{
4717 struct slab_sheaf *empty = NULL;
4718 struct slab_sheaf *full;
4719 struct node_barn *barn;
4720 bool allow_spin;
4721
4722 slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
4723
4724 /* Bootstrap or debug cache, back off */
4725 if (unlikely(!cache_has_sheaves(s))) {
4726 local_unlock(&s->cpu_sheaves->lock);
4727 return NULL;
4728 }
4729
4730 if (pcs->spare && pcs->spare->size > 0) {
4731 swap(pcs->main, pcs->spare);
4732 return pcs;
4733 }
4734
4735 barn = get_barn(s);
4736 if (!barn) {
4737 local_unlock(&s->cpu_sheaves->lock);
4738 return NULL;
4739 }
4740
4741 allow_spin = alloc_flags_allow_spinning(alloc_flags);
4742
4743 full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);
4744
4745 if (full) {
4746 stat(s, BARN_GET);
4747 pcs->main = full;
4748 return pcs;
4749 }
4750
4751 stat(s, BARN_GET_FAIL);
4752
4753 if (allow_spin) {
4754 if (pcs->spare) {
4755 empty = pcs->spare;
4756 pcs->spare = NULL;
4757 } else {
4758 empty = barn_get_empty_sheaf(barn, true);
4759 }
4760 }
4761
4762 local_unlock(&s->cpu_sheaves->lock);
4763 pcs = NULL;
4764
4765 if (!allow_spin)
4766 return NULL;
4767
4768 if (!empty) {
4769 empty = alloc_empty_sheaf(s, gfp, alloc_flags);
4770 if (!empty)
4771 return NULL;
4772 }
4773
4774 if (refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
4775 /*
4776 * we must be very low on memory so don't bother
4777 * with the barn
4778 */
4779 sheaf_flush_unused(s, empty);
4780 free_empty_sheaf(s, empty);
4781
4782 return NULL;
4783 }
4784
4785 full = empty;
4786 empty = NULL;
4787
4788 if (!local_trylock(&s->cpu_sheaves->lock))
4789 goto barn_put;
4790 pcs = this_cpu_ptr(s->cpu_sheaves);
4791
4792 /*
4793 * If we put any empty or full sheaf to the barn below, it's due to
4794 * racing or being migrated to a different cpu. Breaching the barn's
4795 * sheaf limits should be thus rare enough so just ignore them to
4796 * simplify the recovery.
4797 */
4798
4799 if (pcs->main->size == 0) {
4800 if (!pcs->spare)
4801 pcs->spare = pcs->main;
4802 else
4803 barn_put_empty_sheaf(barn, pcs->main);
4804 pcs->main = full;
4805 return pcs;
4806 }
4807
4808 if (!pcs->spare) {
4809 pcs->spare = full;
4810 return pcs;
4811 }
4812
4813 if (pcs->spare->size == 0) {
4814 barn_put_empty_sheaf(barn, pcs->spare);
4815 pcs->spare = full;
4816 return pcs;
4817 }
4818
4819barn_put:
4820 barn_put_full_sheaf(barn, full);
4821 stat(s, BARN_PUT);
4822
4823 return pcs;
4824}
4825
4826static __fastpath_inline
4827void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, int node)
4828{
4829 struct slub_percpu_sheaves *pcs;
4830 bool node_requested;
4831 void *object;
4832
4833 node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
4834
4835 /*
4836 * We assume the percpu sheaves contain only local objects although it's
4837 * not completely guaranteed, so we verify later.
4838 */
4839 if (unlikely(node_requested && node != numa_mem_id())) {
4840 stat(s, ALLOC_NODE_MISMATCH);
4841 return NULL;
4842 }
4843
4844 if (!local_trylock(&s->cpu_sheaves->lock))
4845 return NULL;
4846
4847 pcs = this_cpu_ptr(s->cpu_sheaves);
4848
4849 if (unlikely(pcs->main->size == 0)) {
4850 pcs = __pcs_replace_empty_main(s, pcs, gfp, alloc_flags);
4851 if (unlikely(!pcs))
4852 return NULL;
4853 }
4854
4855 object = pcs->main->objects[pcs->main->size - 1];
4856
4857 if (unlikely(node_requested)) {
4858 /*
4859 * Verify that the object was from the node we want. This could
4860 * be false because of cpu migration during an unlocked part of
4861 * the current allocation or previous freeing process.
4862 */
4863 if (page_to_nid(virt_to_page(object)) != node) {
4864 local_unlock(&s->cpu_sheaves->lock);
4865 stat(s, ALLOC_NODE_MISMATCH);
4866 return NULL;
4867 }
4868 }
4869
4870 pcs->main->size--;
4871
4872 local_unlock(&s->cpu_sheaves->lock);
4873
4874 stat(s, ALLOC_FASTPATH);
4875
4876 return object;
4877}
4878
4879static __fastpath_inline
4880unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
4881{
4882 struct slub_percpu_sheaves *pcs;
4883 struct slab_sheaf *main;
4884 unsigned int allocated = 0;
4885 unsigned int batch;
4886
4887next_batch:
4888 if (!local_trylock(&s->cpu_sheaves->lock))
4889 return allocated;
4890
4891 pcs = this_cpu_ptr(s->cpu_sheaves);
4892
4893 if (unlikely(pcs->main->size == 0)) {
4894
4895 struct slab_sheaf *full;
4896 struct node_barn *barn;
4897
4898 if (unlikely(!cache_has_sheaves(s))) {
4899 local_unlock(&s->cpu_sheaves->lock);
4900 return allocated;
4901 }
4902
4903 if (pcs->spare && pcs->spare->size > 0) {
4904 swap(pcs->main, pcs->spare);
4905 goto do_alloc;
4906 }
4907
4908 barn = get_barn(s);
4909 if (!barn) {
4910 local_unlock(&s->cpu_sheaves->lock);
4911 return allocated;
4912 }
4913
4914 full = barn_replace_empty_sheaf(barn, pcs->main,
4915 /* allow_spin = */ true);
4916
4917 if (full) {
4918 stat(s, BARN_GET);
4919 pcs->main = full;
4920 goto do_alloc;
4921 }
4922
4923 stat(s, BARN_GET_FAIL);
4924
4925 local_unlock(&s->cpu_sheaves->lock);
4926
4927 /*
4928 * Once full sheaves in barn are depleted, let the bulk
4929 * allocation continue from slab pages, otherwise we would just
4930 * be copying arrays of pointers twice.
4931 */
4932 return allocated;
4933 }
4934
4935do_alloc:
4936
4937 main = pcs->main;
4938 batch = min(size, main->size);
4939
4940 main->size -= batch;
4941 memcpy(p, main->objects + main->size, batch * sizeof(void *));
4942
4943 local_unlock(&s->cpu_sheaves->lock);
4944
4945 stat_add(s, ALLOC_FASTPATH, batch);
4946
4947 allocated += batch;
4948
4949 if (batch < size) {
4950 p += batch;
4951 size -= batch;
4952 goto next_batch;
4953 }
4954
4955 return allocated;
4956}
4957
4958
4959/*
4960 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
4961 * have the fastpath folded into their functions. So no function call
4962 * overhead for requests that can be satisfied on the fastpath.
4963 *
4964 * The fastpath works by first checking if the lockless freelist can be used.
4965 * If not then __slab_alloc is called for slow processing.
4966 *
4967 * Otherwise we can simply pick the next object from the lockless free list.
4968 */
4969static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
4970 gfp_t gfpflags, int node, const struct slab_alloc_context *ac)
4971{
4972 void *object;
4973
4974 s = slab_pre_alloc_hook(s, gfpflags);
4975 if (unlikely(!s))
4976 return NULL;
4977
4978 object = kfence_alloc(s, ac->orig_size, gfpflags);
4979 if (unlikely(object))
4980 goto out;
4981
4982 node = apply_strict_numa_policy(node);
4983
4984 object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
4985
4986 if (unlikely(!object))
4987 object = ___slab_alloc(s, gfpflags, node, ac);
4988
4989 maybe_wipe_obj_freeptr(s, object);
4990
4991out:
4992 /*
4993 * In case this fails due to memcg_slab_post_alloc_hook(),
4994 * object is set to NULL
4995 */
4996 slab_post_alloc_hook(s, gfpflags, 1, &object, ac);
4997
4998 return object;
4999}
5000
5001void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)
5002{
5003 void *ret;
5004 const struct slab_alloc_context ac = {
5005 .caller_addr = _RET_IP_,
5006 .orig_size = s->object_size,
5007 .alloc_flags = SLAB_ALLOC_DEFAULT,
5008 };
5009
5010 ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);
5011
5012 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
5013
5014 return ret;
5015}
5016EXPORT_SYMBOL(kmem_cache_alloc_noprof);
5017
5018void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
5019 gfp_t gfpflags)
5020{
5021 void *ret;
5022 const struct slab_alloc_context ac = {
5023 .caller_addr = _RET_IP_,
5024 .orig_size = s->object_size,
5025 .alloc_flags = SLAB_ALLOC_DEFAULT,
5026 .lru = lru,
5027 };
5028
5029 ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);
5030
5031 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
5032
5033 return ret;
5034}
5035EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof);
5036
5037bool kmem_cache_charge(void *objp, gfp_t gfpflags)
5038{
5039 if (!memcg_kmem_online())
5040 return true;
5041
5042 return memcg_slab_post_charge(objp, gfpflags);
5043}
5044EXPORT_SYMBOL(kmem_cache_charge);
5045
5046/**
5047 * kmem_cache_alloc_node - Allocate an object on the specified node
5048 * @s: The cache to allocate from.
5049 * @gfpflags: See kmalloc().
5050 * @node: node number of the target node.
5051 *
5052 * Identical to kmem_cache_alloc but it will allocate memory on the given
5053 * node, which can improve the performance for cpu bound structures.
5054 *
5055 * Fallback to other node is possible if __GFP_THISNODE is not set.
5056 *
5057 * Return: pointer to the new object or %NULL in case of error
5058 */
5059void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)
5060{
5061 void *ret;
5062 const struct slab_alloc_context ac = {
5063 .caller_addr = _RET_IP_,
5064 .orig_size = s->object_size,
5065 .alloc_flags = SLAB_ALLOC_DEFAULT,
5066 };
5067
5068 ret = slab_alloc_node(s, gfpflags, node, &ac);
5069
5070 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node);
5071
5072 return ret;
5073}
5074EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
5075
5076static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
5077 struct slab_sheaf *sheaf, gfp_t gfp)
5078{
5079 gfp_t gfp_nomemalloc;
5080 int ret;
5081
5082 gfp_nomemalloc = gfp | __GFP_NOMEMALLOC;
5083 if (gfp_pfmemalloc_allowed(gfp))
5084 gfp_nomemalloc |= __GFP_NOWARN;
5085
5086 ret = refill_sheaf(s, sheaf, gfp_nomemalloc);
5087
5088 if (likely(!ret || !gfp_pfmemalloc_allowed(gfp)))
5089 return ret;
5090
5091 /*
5092 * if we are allowed to, refill sheaf with pfmemalloc but then remember
5093 * it for when it's returned
5094 */
5095 ret = refill_sheaf(s, sheaf, gfp);
5096 sheaf->pfmemalloc = true;
5097
5098 return ret;
5099}
5100
5101static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
5102 size_t size, void **p);
5103
5104/*
5105 * returns a sheaf that has at least the requested size
5106 * when prefilling is needed, do so with given gfp flags
5107 *
5108 * return NULL if sheaf allocation or prefilling failed
5109 */
5110struct slab_sheaf *
5111kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
5112{
5113 struct slub_percpu_sheaves *pcs;
5114 struct slab_sheaf *sheaf = NULL;
5115 struct node_barn *barn;
5116
5117 if (unlikely(!size))
5118 return NULL;
5119
5120 if (unlikely(size > s->sheaf_capacity)) {
5121
5122 sheaf = __alloc_empty_sheaf(s, gfp, SLAB_ALLOC_DEFAULT, size);
5123 if (!sheaf)
5124 return NULL;
5125
5126 stat(s, SHEAF_PREFILL_OVERSIZE);
5127 sheaf->capacity = size;
5128
5129 /*
5130 * we do not need to care about pfmemalloc here because oversize
5131 * sheaves are always flushed and freed when returned
5132 */
5133 if (!__kmem_cache_alloc_bulk(s, gfp, size,
5134 &sheaf->objects[0])) {
5135 free_empty_sheaf(s, sheaf);
5136 return NULL;
5137 }
5138
5139 sheaf->size = size;
5140
5141 return sheaf;
5142 }
5143
5144 local_lock(&s->cpu_sheaves->lock);
5145 pcs = this_cpu_ptr(s->cpu_sheaves);
5146
5147 if (pcs->spare) {
5148 sheaf = pcs->spare;
5149 pcs->spare = NULL;
5150 stat(s, SHEAF_PREFILL_FAST);
5151 } else {
5152 barn = get_barn(s);
5153
5154 stat(s, SHEAF_PREFILL_SLOW);
5155 if (barn)
5156 sheaf = barn_get_full_or_empty_sheaf(barn);
5157 if (sheaf && sheaf->size)
5158 stat(s, BARN_GET);
5159 else
5160 stat(s, BARN_GET_FAIL);
5161 }
5162
5163 local_unlock(&s->cpu_sheaves->lock);
5164
5165
5166 if (!sheaf)
5167 sheaf = alloc_empty_sheaf(s, gfp, SLAB_ALLOC_DEFAULT);
5168
5169 if (sheaf) {
5170 sheaf->capacity = s->sheaf_capacity;
5171 sheaf->pfmemalloc = false;
5172
5173 if (sheaf->size < size &&
5174 __prefill_sheaf_pfmemalloc(s, sheaf, gfp)) {
5175 sheaf_flush_unused(s, sheaf);
5176 free_empty_sheaf(s, sheaf);
5177 sheaf = NULL;
5178 }
5179 }
5180
5181 return sheaf;
5182}
5183
5184/*
5185 * Use this to return a sheaf obtained by kmem_cache_prefill_sheaf()
5186 *
5187 * If the sheaf cannot simply become the percpu spare sheaf, but there's space
5188 * for a full sheaf in the barn, we try to refill the sheaf back to the cache's
5189 * sheaf_capacity to avoid handling partially full sheaves.
5190 *
5191 * If the refill fails because gfp is e.g. GFP_NOWAIT, or the barn is full, the
5192 * sheaf is instead flushed and freed.
5193 */
5194void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
5195 struct slab_sheaf *sheaf)
5196{
5197 struct slub_percpu_sheaves *pcs;
5198 struct node_barn *barn;
5199
5200 if (unlikely((sheaf->capacity != s->sheaf_capacity)
5201 || sheaf->pfmemalloc)) {
5202 sheaf_flush_unused(s, sheaf);
5203 free_empty_sheaf(s, sheaf);
5204 return;
5205 }
5206
5207 local_lock(&s->cpu_sheaves->lock);
5208 pcs = this_cpu_ptr(s->cpu_sheaves);
5209 barn = get_barn(s);
5210
5211 if (!pcs->spare) {
5212 pcs->spare = sheaf;
5213 sheaf = NULL;
5214 stat(s, SHEAF_RETURN_FAST);
5215 }
5216
5217 local_unlock(&s->cpu_sheaves->lock);
5218
5219 if (!sheaf)
5220 return;
5221
5222 stat(s, SHEAF_RETURN_SLOW);
5223
5224 /*
5225 * If the barn has too many full sheaves or we fail to refill the sheaf,
5226 * simply flush and free it.
5227 */
5228 if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES ||
5229 refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
5230 sheaf_flush_unused(s, sheaf);
5231 free_empty_sheaf(s, sheaf);
5232 return;
5233 }
5234
5235 barn_put_full_sheaf(barn, sheaf);
5236 stat(s, BARN_PUT);
5237}
5238
5239/*
5240 * Refill a sheaf previously returned by kmem_cache_prefill_sheaf to at least
5241 * the given size.
5242 *
5243 * Return: 0 on success. The sheaf will contain at least @size objects.
5244 * The sheaf might have been replaced with a new one if more than
5245 * sheaf->capacity objects are requested.
5246 *
5247 * Return: -ENOMEM on failure. Some objects might have been added to the sheaf
5248 * but the sheaf will not be replaced.
5249 *
5250 * In practice we always refill to full sheaf's capacity.
5251 */
5252int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
5253 struct slab_sheaf **sheafp, unsigned int size)
5254{
5255 struct slab_sheaf *sheaf;
5256
5257 /*
5258 * TODO: do we want to support *sheaf == NULL to be equivalent of
5259 * kmem_cache_prefill_sheaf() ?
5260 */
5261 if (!sheafp || !(*sheafp))
5262 return -EINVAL;
5263
5264 sheaf = *sheafp;
5265 if (sheaf->size >= size)
5266 return 0;
5267
5268 if (likely(sheaf->capacity >= size)) {
5269 if (likely(sheaf->capacity == s->sheaf_capacity))
5270 return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
5271
5272 if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
5273 &sheaf->objects[sheaf->size]))
5274 return -ENOMEM;
5275 sheaf->size = sheaf->capacity;
5276
5277 return 0;
5278 }
5279
5280 /*
5281 * We had a regular sized sheaf and need an oversize one, or we had an
5282 * oversize one already but need a larger one now.
5283 * This should be a very rare path so let's not complicate it.
5284 */
5285 sheaf = kmem_cache_prefill_sheaf(s, gfp, size);
5286 if (!sheaf)
5287 return -ENOMEM;
5288
5289 kmem_cache_return_sheaf(s, gfp, *sheafp);
5290 *sheafp = sheaf;
5291 return 0;
5292}
5293
5294/*
5295 * Allocate from a sheaf obtained by kmem_cache_prefill_sheaf()
5296 *
5297 * Guaranteed not to fail as many allocations as was the requested size.
5298 * After the sheaf is emptied, it fails - no fallback to the slab cache itself.
5299 *
5300 * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT
5301 * memcg charging is forced over limit if necessary, to avoid failure.
5302 *
5303 * It is possible that the allocation comes from kfence and then the sheaf
5304 * size is not decreased.
5305 */
5306void *
5307kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
5308 struct slab_sheaf *sheaf)
5309{
5310 void *ret = NULL;
5311 const struct slab_alloc_context ac = {
5312 .orig_size = s->object_size,
5313 .alloc_flags = SLAB_ALLOC_DEFAULT,
5314 };
5315
5316 if (sheaf->size == 0)
5317 goto out;
5318
5319 ret = kfence_alloc(s, s->object_size, gfp);
5320
5321 if (likely(!ret))
5322 ret = sheaf->objects[--sheaf->size];
5323
5324 /* add __GFP_NOFAIL to force successful memcg charging */
5325 slab_post_alloc_hook(s, gfp | __GFP_NOFAIL, 1, &ret, &ac);
5326out:
5327 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE);
5328
5329 return ret;
5330}
5331
5332unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf)
5333{
5334 return sheaf->size;
5335}
5336/*
5337 * To avoid unnecessary overhead, we pass through large allocation requests
5338 * directly to the page allocator. We use __GFP_COMP, because we will need to
5339 * know the allocation order to free the pages properly in kfree.
5340 */
5341static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
5342{
5343 struct page *page;
5344 void *ptr = NULL;
5345 unsigned int order = get_order(size);
5346
5347 if (unlikely(flags & GFP_SLAB_BUG_MASK))
5348 flags = kmalloc_fix_flags(flags);
5349
5350 flags |= __GFP_COMP;
5351
5352 if (node == NUMA_NO_NODE)
5353 page = alloc_frozen_pages_noprof(flags, order);
5354 else
5355 page = __alloc_frozen_pages_noprof(flags, order, node, NULL,
5356 ALLOC_DEFAULT);
5357
5358 if (page) {
5359 ptr = page_address(page);
5360 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
5361 PAGE_SIZE << order);
5362 __SetPageLargeKmalloc(page);
5363 }
5364
5365 ptr = kasan_kmalloc_large(ptr, size, flags);
5366 /* As ptr might get tagged, call kmemleak hook after KASAN. */
5367 kmemleak_alloc(ptr, size, 1, flags);
5368 kmsan_kmalloc_large(ptr, size, flags);
5369
5370 return ptr;
5371}
5372
5373void *__kmalloc_large_noprof(size_t size, gfp_t flags)
5374{
5375 void *ret = ___kmalloc_large_node(size, flags, NUMA_NO_NODE);
5376
5377 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
5378 flags, NUMA_NO_NODE);
5379 return ret;
5380}
5381EXPORT_SYMBOL(__kmalloc_large_noprof);
5382
5383void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
5384{
5385 void *ret = ___kmalloc_large_node(size, flags, node);
5386
5387 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
5388 flags, node);
5389 return ret;
5390}
5391EXPORT_SYMBOL(__kmalloc_large_node_noprof);
5392
5393static __always_inline
5394void *__do_kmalloc_node(kmem_buckets *b, gfp_t flags, int node,
5395 kmalloc_token_t token, const struct slab_alloc_context *ac)
5396{
5397 const size_t size = ac->orig_size;
5398 struct kmem_cache *s;
5399 void *ret;
5400
5401 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
5402 ret = __kmalloc_large_node_noprof(size, flags, node);
5403 trace_kmalloc(ac->caller_addr, ret, size,
5404 PAGE_SIZE << get_order(size), flags, node);
5405 return ret;
5406 }
5407
5408 if (unlikely(!size))
5409 return ZERO_SIZE_PTR;
5410
5411 s = kmalloc_slab(size, b, flags, token, ac->alloc_flags);
5412
5413 ret = slab_alloc_node(s, flags, node, ac);
5414 ret = kasan_kmalloc(s, ret, size, flags);
5415 trace_kmalloc(ac->caller_addr, ret, size, s->size, flags, node);
5416 return ret;
5417}
5418void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
5419{
5420 const struct slab_alloc_context ac = {
5421 .caller_addr = _RET_IP_,
5422 .orig_size = size,
5423 .alloc_flags = SLAB_ALLOC_DEFAULT,
5424 };
5425
5426 return __do_kmalloc_node(PASS_BUCKET_PARAM(b), flags, node,
5427 PASS_TOKEN_PARAM(token), &ac);
5428}
5429EXPORT_SYMBOL(__kmalloc_node_noprof);
5430
5431void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags)
5432{
5433 const struct slab_alloc_context ac = {
5434 .caller_addr = _RET_IP_,
5435 .orig_size = size,
5436 .alloc_flags = SLAB_ALLOC_DEFAULT,
5437 };
5438
5439 return __do_kmalloc_node(NULL, flags, NUMA_NO_NODE,
5440 PASS_TOKEN_PARAM(token), &ac);
5441}
5442EXPORT_SYMBOL(__kmalloc_noprof);
5443
5444static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags,
5445 int node, const struct slab_alloc_context *ac)
5446{
5447 struct kmem_cache *s;
5448 bool can_retry = true;
5449 void *ret;
5450
5451 VM_WARN_ON_ONCE(alloc_flags_allow_spinning(ac->alloc_flags));
5452 VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO |
5453 __GFP_NOWARN | __GFP_NOMEMALLOC));
5454
5455 gfp_flags |= __GFP_NOWARN | __GFP_NOMEMALLOC;
5456
5457 if (unlikely(!size))
5458 return ZERO_SIZE_PTR;
5459
5460 if (!can_spin_trylock())
5461 return NULL;
5462
5463 node = apply_strict_numa_policy(node);
5464
5465retry:
5466 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
5467 return NULL;
5468
5469 s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token),
5470 ac->alloc_flags);
5471
5472 if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s))
5473 /*
5474 * kmalloc_nolock() is not supported on architectures that
5475 * don't implement cmpxchg16b and thus need slab_lock()
5476 * which could be preempted by a nmi.
5477 * But debug caches don't use that and only rely on
5478 * kmem_cache_node->list_lock, so kmalloc_nolock() can attempt
5479 * to allocate from debug caches by
5480 * spin_trylock_irqsave(&n->list_lock, ...)
5481 */
5482 return NULL;
5483
5484 ret = alloc_from_pcs(s, gfp_flags, ac->alloc_flags, node);
5485 if (ret)
5486 goto success;
5487
5488 /*
5489 * Do not call slab_alloc_node(), since trylock mode isn't
5490 * compatible with slab_pre_alloc_hook/should_failslab and
5491 * kfence_alloc. Hence call ___slab_alloc() (at most twice)
5492 * and slab_post_alloc_hook() directly.
5493 */
5494 ret = ___slab_alloc(s, gfp_flags, node, ac);
5495
5496 /*
5497 * It's possible we failed due to trylock as we preempted someone with
5498 * the sheaves locked, and the list_lock is also held by another cpu.
5499 * But it should be rare that multiple kmalloc buckets would have
5500 * sheaves locked, so try a larger one.
5501 */
5502 if (!ret && can_retry) {
5503 /* pick the next kmalloc bucket */
5504 size = s->object_size + 1;
5505 /*
5506 * Another alternative is to
5507 * if (memcg) gfp_flags &= ~__GFP_ACCOUNT;
5508 * else if (!memcg) gfp_flags |= __GFP_ACCOUNT;
5509 * to retry from bucket of the same size.
5510 */
5511 can_retry = false;
5512 goto retry;
5513 }
5514
5515success:
5516 maybe_wipe_obj_freeptr(s, ret);
5517 slab_post_alloc_hook(s, gfp_flags, 1, &ret, ac);
5518
5519 ret = kasan_kmalloc(s, ret, ac->orig_size, gfp_flags);
5520 return ret;
5521}
5522
5523void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node)
5524{
5525 const struct slab_alloc_context ac = {
5526 .caller_addr = _RET_IP_,
5527 .orig_size = size,
5528 .alloc_flags = SLAB_ALLOC_NOLOCK,
5529 };
5530
5531 return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token),
5532 gfp_flags, node, &ac);
5533}
5534EXPORT_SYMBOL_GPL(_kmalloc_nolock_noprof);
5535
5536void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags,
5537 int node, unsigned long caller)
5538{
5539 const struct slab_alloc_context ac = {
5540 .caller_addr = caller,
5541 .orig_size = size,
5542 .alloc_flags = SLAB_ALLOC_DEFAULT,
5543 };
5544
5545 return __do_kmalloc_node(PASS_BUCKET_PARAM(b), flags, node,
5546 PASS_TOKEN_PARAM(token), &ac);
5547}
5548EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof);
5549
5550void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
5551{
5552 void *ret;
5553 const struct slab_alloc_context ac = {
5554 .caller_addr = _RET_IP_,
5555 .orig_size = size,
5556 .alloc_flags = SLAB_ALLOC_DEFAULT,
5557 };
5558
5559 ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);
5560
5561 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
5562
5563 ret = kasan_kmalloc(s, ret, size, gfpflags);
5564 return ret;
5565}
5566EXPORT_SYMBOL(__kmalloc_cache_noprof);
5567
5568void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
5569 int node, size_t size)
5570{
5571 void *ret;
5572 const struct slab_alloc_context ac = {
5573 .caller_addr = _RET_IP_,
5574 .orig_size = size,
5575 .alloc_flags = SLAB_ALLOC_DEFAULT,
5576 };
5577
5578 ret = slab_alloc_node(s, gfpflags, node, &ac);
5579
5580 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
5581
5582 ret = kasan_kmalloc(s, ret, size, gfpflags);
5583 return ret;
5584}
5585EXPORT_SYMBOL(__kmalloc_cache_node_noprof);
5586
5587/*
5588 * The only version of kmalloc_node() that takes alloc_flags and thus can
5589 * determine on its own whether to handle the allocation via kmalloc_nolock() or
5590 * normally
5591 */
5592void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags,
5593 unsigned int alloc_flags, int node)
5594{
5595 const struct slab_alloc_context ac = {
5596 .caller_addr = _RET_IP_,
5597 .orig_size = size,
5598 .alloc_flags = alloc_flags,
5599 };
5600
5601 if (alloc_flags_allow_spinning(alloc_flags)) {
5602 return __do_kmalloc_node(NULL, flags, node,
5603 PASS_TOKEN_PARAM(token), &ac);
5604 } else {
5605 return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token),
5606 flags, node, &ac);
5607 }
5608}
5609
5610
5611static noinline void free_to_partial_list(
5612 struct kmem_cache *s, struct slab *slab,
5613 void *head, void *tail, int bulk_cnt,
5614 unsigned long addr)
5615{
5616 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
5617 struct slab *slab_free = NULL;
5618 int cnt = bulk_cnt;
5619 unsigned long flags;
5620 depot_stack_handle_t handle = 0;
5621
5622 /*
5623 * We cannot use GFP_NOWAIT as there are callsites where waking up
5624 * kswapd could deadlock
5625 */
5626 if (s->flags & SLAB_STORE_USER)
5627 handle = set_track_prepare(__GFP_NOWARN);
5628
5629 spin_lock_irqsave(&n->list_lock, flags);
5630
5631 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) {
5632 void *prior = slab->freelist;
5633
5634 /* Perform the actual freeing while we still hold the locks */
5635 slab->inuse -= cnt;
5636 set_freepointer(s, tail, prior);
5637 slab->freelist = head;
5638
5639 /*
5640 * If the slab is empty, and node's partial list is full,
5641 * it should be discarded anyway no matter it's on full or
5642 * partial list.
5643 */
5644 if (slab->inuse == 0 && n->nr_partial >= s->min_partial)
5645 slab_free = slab;
5646
5647 if (!prior) {
5648 /* was on full list */
5649 remove_full(s, n, slab);
5650 if (!slab_free) {
5651 add_partial(n, slab, ADD_TO_TAIL);
5652 stat(s, FREE_ADD_PARTIAL);
5653 }
5654 } else if (slab_free) {
5655 remove_partial(n, slab);
5656 stat(s, FREE_REMOVE_PARTIAL);
5657 }
5658 }
5659
5660 if (slab_free) {
5661 /*
5662 * Update the counters while still holding n->list_lock to
5663 * prevent spurious validation warnings
5664 */
5665 dec_slabs_node(s, slab_nid(slab_free), slab_free->objects);
5666 }
5667
5668 spin_unlock_irqrestore(&n->list_lock, flags);
5669
5670 if (slab_free) {
5671 stat(s, FREE_SLAB);
5672 free_slab(s, slab_free);
5673 }
5674}
5675
5676/*
5677 * Try returning (remainder of) the freelist that we just detached from the
5678 * slab. Optimistically assume the slab is still full, so we don't need to find
5679 * the tail of the detached freelist.
5680 *
5681 * Fail if the slab isn't full anymore due to a concurrent free.
5682 */
5683static bool __slab_try_return_freelist(struct kmem_cache *s,
5684 struct kmem_cache_node *n,
5685 struct slab *slab, void *head, int cnt)
5686{
5687 struct freelist_counters old, new;
5688 unsigned long flags;
5689
5690 old.freelist = slab->freelist;
5691 old.counters = slab->counters;
5692
5693 if (old.freelist)
5694 return false;
5695
5696 new.freelist = head;
5697 new.counters = old.counters;
5698 new.inuse -= cnt;
5699
5700 spin_lock_irqsave(&n->list_lock, flags);
5701
5702 if (!slab_update_freelist(s, slab, &old, &new, "__slab_try_return_freelist")) {
5703 spin_unlock_irqrestore(&n->list_lock, flags);
5704 return false;
5705 }
5706
5707 add_partial(n, slab, ADD_TO_TAIL);
5708 spin_unlock_irqrestore(&n->list_lock, flags);
5709 return true;
5710}
5711
5712/*
5713 * Slow path handling. This may still be called frequently since objects
5714 * have a longer lifetime than the cpu slabs in most processing loads.
5715 *
5716 * So we still attempt to reduce cache line usage. Just take the slab
5717 * lock and free the item. If there is no additional partial slab
5718 * handling required then we can return immediately.
5719 */
5720static void __slab_free(struct kmem_cache *s, struct slab *slab,
5721 void *head, void *tail, int cnt,
5722 unsigned long addr)
5723
5724{
5725 bool was_full;
5726 struct freelist_counters old, new;
5727 struct kmem_cache_node *n = NULL;
5728 unsigned long flags;
5729 bool on_node_partial;
5730
5731 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
5732 free_to_partial_list(s, slab, head, tail, cnt, addr);
5733 return;
5734 }
5735
5736 do {
5737 if (unlikely(n)) {
5738 spin_unlock_irqrestore(&n->list_lock, flags);
5739 n = NULL;
5740 }
5741
5742 old.freelist = slab->freelist;
5743 old.counters = slab->counters;
5744
5745 was_full = (old.freelist == NULL);
5746
5747 set_freepointer(s, tail, old.freelist);
5748
5749 new.freelist = head;
5750 new.counters = old.counters;
5751 new.inuse -= cnt;
5752
5753 /*
5754 * Might need to be taken off (due to becoming empty) or added
5755 * to (due to not being full anymore) the partial list.
5756 * Unless it's frozen.
5757 */
5758 if (!new.inuse || was_full) {
5759
5760 n = get_node(s, slab_nid(slab));
5761 /*
5762 * Speculatively acquire the list_lock.
5763 * If the cmpxchg does not succeed then we may
5764 * drop the list_lock without any processing.
5765 *
5766 * Otherwise the list_lock will synchronize with
5767 * other processors updating the list of slabs.
5768 */
5769 spin_lock_irqsave(&n->list_lock, flags);
5770
5771 on_node_partial = slab_test_node_partial(slab);
5772 }
5773
5774 } while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));
5775
5776 if (likely(!n)) {
5777 /*
5778 * We didn't take the list_lock because the slab was already on
5779 * the partial list and will remain there.
5780 */
5781 return;
5782 }
5783
5784 /*
5785 * This slab was partially empty but not on the per-node partial list,
5786 * in which case we shouldn't manipulate its list, just return.
5787 */
5788 if (!was_full && !on_node_partial) {
5789 spin_unlock_irqrestore(&n->list_lock, flags);
5790 return;
5791 }
5792
5793 /*
5794 * If slab became empty, should we add/keep it on the partial list or we
5795 * have enough?
5796 */
5797 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
5798 goto slab_empty;
5799
5800 /*
5801 * Objects left in the slab. If it was not on the partial list before
5802 * then add it.
5803 */
5804 if (unlikely(was_full)) {
5805 add_partial(n, slab, ADD_TO_TAIL);
5806 stat(s, FREE_ADD_PARTIAL);
5807 }
5808 spin_unlock_irqrestore(&n->list_lock, flags);
5809 return;
5810
5811slab_empty:
5812 /*
5813 * The slab could have a single object and thus go from full to empty in
5814 * a single free, but more likely it was on the partial list. Remove it.
5815 */
5816 if (likely(!was_full)) {
5817 remove_partial(n, slab);
5818 stat(s, FREE_REMOVE_PARTIAL);
5819 }
5820
5821 spin_unlock_irqrestore(&n->list_lock, flags);
5822 stat(s, FREE_SLAB);
5823 discard_slab(s, slab);
5824}
5825
5826/*
5827 * pcs is locked. We should have get rid of the spare sheaf and obtained an
5828 * empty sheaf, while the main sheaf is full. We want to install the empty sheaf
5829 * as a main sheaf, and make the current main sheaf a spare sheaf.
5830 *
5831 * However due to having relinquished the cpu_sheaves lock when obtaining
5832 * the empty sheaf, we need to handle some unlikely but possible cases.
5833 *
5834 * If we put any sheaf to barn here, it's because we were interrupted or have
5835 * been migrated to a different cpu, which should be rare enough so just ignore
5836 * the barn's limits to simplify the handling.
5837 *
5838 * An alternative scenario that gets us here is when we fail
5839 * barn_replace_full_sheaf(), because there's no empty sheaf available in the
5840 * barn, so we had to allocate it by alloc_empty_sheaf(). But because we saw the
5841 * limit on full sheaves was not exceeded, we assume it didn't change and just
5842 * put the full sheaf there.
5843 */
5844static void __pcs_install_empty_sheaf(struct kmem_cache *s,
5845 struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty,
5846 struct node_barn *barn)
5847{
5848 slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
5849
5850 /* This is what we expect to find if nobody interrupted us. */
5851 if (likely(!pcs->spare)) {
5852 pcs->spare = pcs->main;
5853 pcs->main = empty;
5854 return;
5855 }
5856
5857 /*
5858 * Unlikely because if the main sheaf had space, we would have just
5859 * freed to it. Get rid of our empty sheaf.
5860 */
5861 if (pcs->main->size < s->sheaf_capacity) {
5862 barn_put_empty_sheaf(barn, empty);
5863 return;
5864 }
5865
5866 /* Also unlikely for the same reason */
5867 if (pcs->spare->size < s->sheaf_capacity) {
5868 swap(pcs->main, pcs->spare);
5869 barn_put_empty_sheaf(barn, empty);
5870 return;
5871 }
5872
5873 /*
5874 * We probably failed barn_replace_full_sheaf() due to no empty sheaf
5875 * available there, but we allocated one, so finish the job.
5876 */
5877 barn_put_full_sheaf(barn, pcs->main);
5878 stat(s, BARN_PUT);
5879 pcs->main = empty;
5880}
5881
5882/*
5883 * Replace the full main sheaf with a (at least partially) empty sheaf.
5884 *
5885 * Must be called with the cpu_sheaves local lock locked. If successful, returns
5886 * the pcs pointer and the local lock locked (possibly on a different cpu than
5887 * initially called). If not successful, returns NULL and the local lock
5888 * unlocked.
5889 */
5890static struct slub_percpu_sheaves *
5891__pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
5892 bool allow_spin)
5893{
5894 struct slab_sheaf *empty;
5895 struct node_barn *barn;
5896 bool put_fail;
5897
5898restart:
5899 slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
5900
5901 /* Bootstrap or debug cache, back off */
5902 if (unlikely(!cache_has_sheaves(s))) {
5903 local_unlock(&s->cpu_sheaves->lock);
5904 return NULL;
5905 }
5906
5907 barn = get_barn(s);
5908 if (!barn) {
5909 local_unlock(&s->cpu_sheaves->lock);
5910 return NULL;
5911 }
5912
5913 put_fail = false;
5914
5915 if (!pcs->spare) {
5916 empty = barn_get_empty_sheaf(barn, allow_spin);
5917 if (empty) {
5918 pcs->spare = pcs->main;
5919 pcs->main = empty;
5920 return pcs;
5921 }
5922 goto alloc_empty;
5923 }
5924
5925 if (pcs->spare->size < s->sheaf_capacity) {
5926 swap(pcs->main, pcs->spare);
5927 return pcs;
5928 }
5929
5930 empty = barn_replace_full_sheaf(barn, pcs->main, allow_spin);
5931
5932 if (!IS_ERR(empty)) {
5933 stat(s, BARN_PUT);
5934 pcs->main = empty;
5935 return pcs;
5936 }
5937
5938 /* sheaf_flush_unused() doesn't support !allow_spin */
5939 if (PTR_ERR(empty) == -E2BIG && allow_spin) {
5940 /* Since we got here, spare exists and is full */
5941 struct slab_sheaf *to_flush = pcs->spare;
5942
5943 stat(s, BARN_PUT_FAIL);
5944
5945 pcs->spare = NULL;
5946 local_unlock(&s->cpu_sheaves->lock);
5947
5948 sheaf_flush_unused(s, to_flush);
5949 empty = to_flush;
5950 goto got_empty;
5951 }
5952
5953 /*
5954 * We could not replace full sheaf because barn had no empty
5955 * sheaves. We can still allocate it and put the full sheaf in
5956 * __pcs_install_empty_sheaf(), but if we fail to allocate it,
5957 * make sure to count the fail.
5958 */
5959 put_fail = true;
5960
5961alloc_empty:
5962 local_unlock(&s->cpu_sheaves->lock);
5963
5964 /*
5965 * alloc_empty_sheaf() doesn't support !allow_spin and it's
5966 * easier to fall back to freeing directly without sheaves
5967 * than add the support (and to sheaf_flush_unused() above)
5968 */
5969 if (!allow_spin)
5970 return NULL;
5971
5972 /* Don't wake up kswapd, it will cause deadlock under pi_lock */
5973 empty = alloc_empty_sheaf(s, __GFP_NOWARN, SLAB_ALLOC_DEFAULT);
5974 if (empty)
5975 goto got_empty;
5976
5977 if (put_fail)
5978 stat(s, BARN_PUT_FAIL);
5979
5980 if (!sheaf_try_flush_main(s))
5981 return NULL;
5982
5983 if (!local_trylock(&s->cpu_sheaves->lock))
5984 return NULL;
5985
5986 pcs = this_cpu_ptr(s->cpu_sheaves);
5987
5988 /*
5989 * we flushed the main sheaf so it should be empty now,
5990 * but in case we got preempted or migrated, we need to
5991 * check again
5992 */
5993 if (pcs->main->size == s->sheaf_capacity)
5994 goto restart;
5995
5996 return pcs;
5997
5998got_empty:
5999 if (!local_trylock(&s->cpu_sheaves->lock)) {
6000 barn_put_empty_sheaf(barn, empty);
6001 return NULL;
6002 }
6003
6004 pcs = this_cpu_ptr(s->cpu_sheaves);
6005 __pcs_install_empty_sheaf(s, pcs, empty, barn);
6006
6007 return pcs;
6008}
6009
6010/*
6011 * Free an object to the percpu sheaves.
6012 * The object is expected to have passed slab_free_hook() already.
6013 */
6014static __fastpath_inline
6015bool free_to_pcs(struct kmem_cache *s, void *object, bool allow_spin)
6016{
6017 struct slub_percpu_sheaves *pcs;
6018
6019 if (!local_trylock(&s->cpu_sheaves->lock))
6020 return false;
6021
6022 pcs = this_cpu_ptr(s->cpu_sheaves);
6023
6024 if (unlikely(pcs->main->size == s->sheaf_capacity)) {
6025
6026 pcs = __pcs_replace_full_main(s, pcs, allow_spin);
6027 if (unlikely(!pcs))
6028 return false;
6029 }
6030
6031 pcs->main->objects[pcs->main->size++] = object;
6032
6033 local_unlock(&s->cpu_sheaves->lock);
6034
6035 stat(s, FREE_FASTPATH);
6036
6037 return true;
6038}
6039
6040static void rcu_free_sheaf(struct rcu_head *head)
6041{
6042 struct slab_sheaf *sheaf;
6043 struct node_barn *barn = NULL;
6044 struct kmem_cache *s;
6045
6046 sheaf = container_of(head, struct slab_sheaf, rcu_head);
6047
6048 s = sheaf->cache;
6049
6050 /*
6051 * This may remove some objects due to slab_free_hook() returning false,
6052 * so that the sheaf might no longer be completely full. But it's easier
6053 * to handle it as full (unless it became completely empty), as the code
6054 * handles it fine. The only downside is that sheaf will serve fewer
6055 * allocations when reused. It only happens due to debugging, which is a
6056 * performance hit anyway.
6057 *
6058 * If it returns true, there was at least one object from pfmemalloc
6059 * slab so simply flush everything.
6060 */
6061 if (__rcu_free_sheaf_prepare(s, sheaf))
6062 goto flush;
6063
6064 barn = get_barn_node(s, sheaf->node);
6065 if (!barn)
6066 goto flush;
6067
6068 /* due to slab_free_hook() */
6069 if (unlikely(sheaf->size == 0))
6070 goto empty;
6071
6072 /*
6073 * Checking nr_full/nr_empty outside lock avoids contention in case the
6074 * barn is at the respective limit. Due to the race we might go over the
6075 * limit but that should be rare and harmless.
6076 */
6077
6078 if (data_race(barn->nr_full) < MAX_FULL_SHEAVES) {
6079 stat(s, BARN_PUT);
6080 barn_put_full_sheaf(barn, sheaf);
6081 return;
6082 }
6083
6084flush:
6085 stat(s, BARN_PUT_FAIL);
6086 sheaf_flush_unused(s, sheaf);
6087
6088empty:
6089 if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) {
6090 barn_put_empty_sheaf(barn, sheaf);
6091 return;
6092 }
6093
6094 free_empty_sheaf(s, sheaf);
6095}
6096
6097/*
6098 * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since
6099 * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT),
6100 * this would violate lock nesting rules. Therefore, kfree_call_rcu_nolock()
6101 * avoids this problem by passing SLAB_FREE_NOLOCK. kvfree_call_rcu() is
6102 * bypassing the sheaves layer completely on PREEMPT_RT.
6103 *
6104 * However, lockdep still complains that it is invalid to acquire spinlock_t
6105 * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a
6106 * spinning lock. Tell lockdep that acquiring spinlock_t is valid here
6107 * by temporarily raising the wait-type to LD_WAIT_CONFIG. Skip the lockdep map
6108 * on PREEMPT_RT to avoid suppressing valid lockdep warnings.
6109 */
6110static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG);
6111
6112bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
6113{
6114 struct slub_percpu_sheaves *pcs;
6115 struct slab_sheaf *rcu_sheaf;
6116 bool allow_spin = free_flags_allow_spinning(free_flags);
6117
6118 VM_WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT) && allow_spin);
6119
6120 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6121 lock_map_acquire_try(&kfree_rcu_sheaf_map);
6122
6123 if (!local_trylock(&s->cpu_sheaves->lock))
6124 goto fail;
6125
6126 pcs = this_cpu_ptr(s->cpu_sheaves);
6127
6128 if (unlikely(!pcs->rcu_free)) {
6129 struct slab_sheaf *empty;
6130 struct node_barn *barn;
6131 unsigned int alloc_flags = to_alloc_flags(free_flags);
6132
6133 /* Bootstrap or debug cache, fall back */
6134 if (unlikely(!cache_has_sheaves(s))) {
6135 local_unlock(&s->cpu_sheaves->lock);
6136 goto fail;
6137 }
6138
6139 if (pcs->spare && pcs->spare->size == 0) {
6140 pcs->rcu_free = pcs->spare;
6141 pcs->spare = NULL;
6142 goto do_free;
6143 }
6144
6145 barn = get_barn(s);
6146 if (!barn) {
6147 local_unlock(&s->cpu_sheaves->lock);
6148 goto fail;
6149 }
6150
6151 empty = barn_get_empty_sheaf(barn, allow_spin);
6152
6153 if (empty) {
6154 pcs->rcu_free = empty;
6155 goto do_free;
6156 }
6157
6158 local_unlock(&s->cpu_sheaves->lock);
6159
6160 /* Don't wake up kswapd, it will cause deadlock under pi_lock */
6161 empty = alloc_empty_sheaf(s, __GFP_NOWARN, alloc_flags);
6162
6163 if (!empty)
6164 goto fail;
6165
6166 if (!local_trylock(&s->cpu_sheaves->lock)) {
6167 __free_empty_sheaf(s, empty, free_flags);
6168 goto fail;
6169 }
6170
6171 pcs = this_cpu_ptr(s->cpu_sheaves);
6172
6173 if (unlikely(pcs->rcu_free))
6174 __free_empty_sheaf(s, empty, free_flags);
6175 else
6176 pcs->rcu_free = empty;
6177 }
6178
6179do_free:
6180
6181 rcu_sheaf = pcs->rcu_free;
6182
6183 /*
6184 * Since we flush immediately when size reaches capacity, we never reach
6185 * this with size already at capacity, so no OOB write is possible.
6186 */
6187 rcu_sheaf->objects[rcu_sheaf->size++] = obj;
6188
6189 if (likely(rcu_sheaf->size < s->sheaf_capacity)) {
6190 rcu_sheaf = NULL;
6191 } else {
6192 pcs->rcu_free = NULL;
6193 rcu_sheaf->node = numa_node_id();
6194 }
6195
6196 /*
6197 * we flush before local_unlock to make sure a racing
6198 * flush_all_rcu_sheaves() doesn't miss this sheaf
6199 */
6200 if (rcu_sheaf) {
6201 /*
6202 * With !allow_spin, we might have interrupted call_rcu()'s
6203 * IRQ-disabled critical section. If IRQs are not disabled,
6204 * we know that's not the case.
6205 */
6206 if (unlikely(!allow_spin && irqs_disabled())) {
6207 struct deferred_percpu_work *dpw;
6208
6209 dpw = this_cpu_ptr(&deferred_percpu_work);
6210 if (llist_add(&rcu_sheaf->llnode, &dpw->rcu_sheaves))
6211 irq_work_queue(&dpw->work);
6212 } else {
6213 call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf);
6214 }
6215 }
6216
6217 local_unlock(&s->cpu_sheaves->lock);
6218
6219 stat(s, FREE_RCU_SHEAF);
6220 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6221 lock_map_release(&kfree_rcu_sheaf_map);
6222 return true;
6223
6224fail:
6225 stat(s, FREE_RCU_SHEAF_FAIL);
6226 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6227 lock_map_release(&kfree_rcu_sheaf_map);
6228 return false;
6229}
6230
6231static __always_inline bool can_free_to_pcs(struct slab *slab)
6232{
6233 int slab_node;
6234 int numa_node;
6235
6236 if (!IS_ENABLED(CONFIG_NUMA))
6237 goto check_pfmemalloc;
6238
6239 slab_node = slab_nid(slab);
6240
6241#ifdef CONFIG_HAVE_MEMORYLESS_NODES
6242 /*
6243 * numa_mem_id() points to the closest node with memory so only allow
6244 * objects from that node to the percpu sheaves
6245 */
6246 numa_node = numa_mem_id();
6247
6248 if (likely(slab_node == numa_node))
6249 goto check_pfmemalloc;
6250#else
6251
6252 /*
6253 * numa_mem_id() is only a wrapper to numa_node_id() which is where this
6254 * cpu belongs to, but it might be a memoryless node anyway. We don't
6255 * know what the closest node is.
6256 */
6257 numa_node = numa_node_id();
6258
6259 /* freed object is from this cpu's node, proceed */
6260 if (likely(slab_node == numa_node))
6261 goto check_pfmemalloc;
6262
6263 /*
6264 * Freed object isn't from this cpu's node, but that node is memoryless
6265 * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
6266 * Proceed as it's better to cache remote objects than falling back to
6267 * the slowpath for everything. The allocation side can never obtain
6268 * a local object anyway, if none exist. We don't have numa_mem_id() to
6269 * point to the closest node as we would on a proper memoryless node
6270 * setup.
6271 */
6272 if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
6273 goto check_pfmemalloc;
6274#endif
6275
6276 return false;
6277
6278check_pfmemalloc:
6279 return likely(!slab_test_pfmemalloc(slab));
6280}
6281
6282/*
6283 * Try to free as many objects (already processed by free hooks) as possible to
6284 * a single per-cpu sheaf.
6285 *
6286 * Returns how many objects were freed. Zero means failure and the caller should
6287 * fall back to __kmem_cache_free_bulk().
6288 */
6289static unsigned int __free_to_pcs_batch(struct kmem_cache *s, size_t size, void **p)
6290{
6291 struct slub_percpu_sheaves *pcs;
6292 struct slab_sheaf *main, *empty;
6293 struct node_barn *barn;
6294 unsigned int batch;
6295
6296 if (!local_trylock(&s->cpu_sheaves->lock))
6297 return 0;
6298
6299 pcs = this_cpu_ptr(s->cpu_sheaves);
6300
6301 if (likely(pcs->main->size < s->sheaf_capacity))
6302 goto do_free;
6303
6304 barn = get_barn(s);
6305 if (!barn)
6306 goto no_empty;
6307
6308 if (!pcs->spare) {
6309 empty = barn_get_empty_sheaf(barn, true);
6310 if (!empty)
6311 goto no_empty;
6312
6313 pcs->spare = pcs->main;
6314 pcs->main = empty;
6315 goto do_free;
6316 }
6317
6318 if (pcs->spare->size < s->sheaf_capacity) {
6319 swap(pcs->main, pcs->spare);
6320 goto do_free;
6321 }
6322
6323 empty = barn_replace_full_sheaf(barn, pcs->main, true);
6324 if (IS_ERR(empty)) {
6325 stat(s, BARN_PUT_FAIL);
6326 goto no_empty;
6327 }
6328
6329 stat(s, BARN_PUT);
6330 pcs->main = empty;
6331
6332do_free:
6333 main = pcs->main;
6334 batch = min(size, s->sheaf_capacity - main->size);
6335
6336 memcpy(main->objects + main->size, p, batch * sizeof(void *));
6337 main->size += batch;
6338
6339 local_unlock(&s->cpu_sheaves->lock);
6340
6341 stat_add(s, FREE_FASTPATH, batch);
6342
6343 return batch;
6344
6345no_empty:
6346 local_unlock(&s->cpu_sheaves->lock);
6347
6348 return 0;
6349}
6350
6351/*
6352 * Bulk free objects to the percpu sheaves.
6353 * Unlike free_to_pcs() this includes the calls to all necessary hooks
6354 * and the fallback to freeing to slab pages.
6355 */
6356static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
6357{
6358 bool init = slab_want_init_on_free(s);
6359 void **remote_objects = p;
6360 unsigned int remote_nr = 0;
6361
6362 /*
6363 * Process the free hooks and separate out remote objects by
6364 * partitioning the 'p' array in place:
6365 *
6366 * [0, remote_nr) - processed remote objects
6367 * [remote_nr, i) - processed local objects
6368 * [i, size) - unprocessed objects
6369 */
6370 for (unsigned int i = 0; i < size;) {
6371 struct slab *slab = virt_to_slab(p[i]);
6372
6373 memcg_slab_free_hook(s, slab, p + i, 1);
6374 alloc_tagging_slab_free_hook(s, slab, p + i, 1);
6375
6376 if (unlikely(!slab_free_hook(s, p[i], init, false))) {
6377 p[i] = p[--size];
6378 continue;
6379 }
6380
6381 if (unlikely(!can_free_to_pcs(slab))) {
6382 if (i != remote_nr)
6383 swap(remote_objects[remote_nr], p[i]);
6384 remote_nr++;
6385 }
6386
6387 i++;
6388 }
6389
6390 p += remote_nr;
6391 size -= remote_nr;
6392
6393 while (size) {
6394 unsigned int batch_freed = __free_to_pcs_batch(s, size, p);
6395
6396 if (!batch_freed) {
6397 __kmem_cache_free_bulk(s, size, p);
6398 stat_add(s, FREE_SLOWPATH, size);
6399 break;
6400 }
6401
6402 p += batch_freed;
6403 size -= batch_freed;
6404 }
6405
6406 /*
6407 * Processing remote objects last decreases the chances of cpu migration
6408 * while freeing to sheaves and compromising object locality
6409 */
6410 if (remote_nr) {
6411 __kmem_cache_free_bulk(s, remote_nr, remote_objects);
6412 stat_add(s, FREE_SLOWPATH, remote_nr);
6413 }
6414}
6415
6416/*
6417 * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe
6418 * to take sleeping spin_locks from __slab_free().
6419 * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore().
6420 */
6421static void deferred_percpu_work_fn(struct irq_work *work)
6422{
6423 struct deferred_percpu_work *dpw;
6424 struct llist_head *objs, *objs_by_rcu, *rcu_sheaves;
6425 struct llist_node *llnode, *pos, *t;
6426 struct slab_sheaf *sheaf, *next;
6427
6428 dpw = container_of(work, struct deferred_percpu_work, work);
6429 rcu_sheaves = &dpw->rcu_sheaves;
6430 objs = &dpw->objects;
6431 objs_by_rcu = &dpw->objects_by_rcu;
6432
6433 llnode = llist_del_all(objs);
6434 llist_for_each_safe(pos, t, llnode) {
6435 struct kmem_cache *s;
6436 struct slab *slab;
6437 void *x = pos;
6438
6439 slab = virt_to_slab(x);
6440 s = slab->slab_cache;
6441
6442 /* Point 'x' back to the beginning of allocated object */
6443 x -= s->offset;
6444
6445 /*
6446 * We used freepointer in 'x' to link 'x' into df->objects.
6447 * Clear it to NULL to avoid false positive detection
6448 * of "Freepointer corruption".
6449 */
6450 set_freepointer(s, x, NULL);
6451
6452 __slab_free(s, slab, x, x, 1, _THIS_IP_);
6453 stat(s, FREE_SLOWPATH);
6454 }
6455
6456 llnode = llist_del_all(objs_by_rcu);
6457 llist_for_each_safe(pos, t, llnode) {
6458 void *head = pos;
6459 void *objp = kvmalloc_obj_start_addr(head);
6460
6461 kvfree_call_rcu(head, objp);
6462 }
6463
6464 llnode = llist_del_all(rcu_sheaves);
6465 llist_for_each_entry_safe(sheaf, next, llnode, llnode)
6466 call_rcu(&sheaf->rcu_head, rcu_free_sheaf);
6467}
6468
6469static void defer_free(struct kmem_cache *s, void *head)
6470{
6471 struct deferred_percpu_work *dpw;
6472
6473 guard(preempt)();
6474
6475 head = kasan_reset_tag(head);
6476
6477 dpw = this_cpu_ptr(&deferred_percpu_work);
6478 if (llist_add(head + s->offset, &dpw->objects))
6479 irq_work_queue(&dpw->work);
6480}
6481
6482void defer_kfree_rcu(struct kvfree_rcu_head *head)
6483{
6484 struct deferred_percpu_work *dpw;
6485
6486 guard(preempt)();
6487
6488 dpw = this_cpu_ptr(&deferred_percpu_work);
6489 if (llist_add((struct llist_node *)head, &dpw->objects_by_rcu))
6490 irq_work_queue(&dpw->work);
6491}
6492
6493/* Must be called before flush_rcu_sheaves_on_cache() */
6494void deferred_work_barrier(void)
6495{
6496 int cpu;
6497
6498 for_each_possible_cpu(cpu)
6499 irq_work_sync(&per_cpu_ptr(&deferred_percpu_work, cpu)->work);
6500}
6501
6502static __fastpath_inline
6503void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
6504 unsigned long addr)
6505{
6506 memcg_slab_free_hook(s, slab, &object, 1);
6507 alloc_tagging_slab_free_hook(s, slab, &object, 1);
6508
6509 if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false)))
6510 return;
6511
6512 if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true)))
6513 return;
6514
6515 __slab_free(s, slab, object, object, 1, addr);
6516 stat(s, FREE_SLOWPATH);
6517}
6518
6519#ifdef CONFIG_MEMCG
6520/* Do not inline the rare memcg charging failed path into the allocation path */
6521static noinline
6522void memcg_alloc_abort_single(struct kmem_cache *s, void *object)
6523{
6524 struct slab *slab = virt_to_slab(object);
6525
6526 alloc_tagging_slab_free_hook(s, slab, &object, 1);
6527
6528 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false)))
6529 __slab_free(s, slab, object, object, 1, _RET_IP_);
6530}
6531#endif
6532
6533static __fastpath_inline
6534void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
6535 void *tail, void **p, int cnt, unsigned long addr)
6536{
6537 memcg_slab_free_hook(s, slab, p, cnt);
6538 alloc_tagging_slab_free_hook(s, slab, p, cnt);
6539 /*
6540 * With KASAN enabled slab_free_freelist_hook modifies the freelist
6541 * to remove objects, whose reuse must be delayed.
6542 */
6543 if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) {
6544 __slab_free(s, slab, head, tail, cnt, addr);
6545 stat_add(s, FREE_SLOWPATH, cnt);
6546 }
6547}
6548
6549#ifdef CONFIG_SLUB_RCU_DEBUG
6550static void slab_free_after_rcu_debug(struct rcu_head *rcu_head)
6551{
6552 struct rcu_delayed_free *delayed_free =
6553 container_of(rcu_head, struct rcu_delayed_free, head);
6554 void *object = delayed_free->object;
6555 struct slab *slab = virt_to_slab(object);
6556 struct kmem_cache *s;
6557
6558 kfree(delayed_free);
6559
6560 if (WARN_ON(is_kfence_address(object)))
6561 return;
6562
6563 /* find the object and the cache again */
6564 if (WARN_ON(!slab))
6565 return;
6566 s = slab->slab_cache;
6567 if (WARN_ON(!(s->flags & SLAB_TYPESAFE_BY_RCU)))
6568 return;
6569
6570 /* resume freeing */
6571 if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) {
6572 __slab_free(s, slab, object, object, 1, _THIS_IP_);
6573 stat(s, FREE_SLOWPATH);
6574 }
6575}
6576#endif /* CONFIG_SLUB_RCU_DEBUG */
6577
6578#ifdef CONFIG_KASAN_GENERIC
6579void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
6580{
6581 __slab_free(cache, virt_to_slab(x), x, x, 1, addr);
6582 stat(cache, FREE_SLOWPATH);
6583}
6584#endif
6585
6586static noinline void warn_free_bad_obj(struct kmem_cache *s, void *obj)
6587{
6588 struct kmem_cache *cachep;
6589 struct slab *slab;
6590
6591 slab = virt_to_slab(obj);
6592 if (WARN_ONCE(!slab,
6593 "kmem_cache_free(%s, %p): object is not in a slab page\n",
6594 s->name, obj))
6595 return;
6596
6597 cachep = slab->slab_cache;
6598
6599 if (WARN_ONCE(cachep != s,
6600 "kmem_cache_free(%s, %p): object belongs to different cache %s\n",
6601 s->name, obj, cachep ? cachep->name : "(NULL)")) {
6602 if (cachep)
6603 print_tracking(cachep, obj);
6604 return;
6605 }
6606}
6607
6608/**
6609 * kmem_cache_free - Deallocate an object
6610 * @s: The cache the allocation was from.
6611 * @x: The previously allocated object.
6612 *
6613 * Free an object which was previously allocated from this
6614 * cache.
6615 */
6616void kmem_cache_free(struct kmem_cache *s, void *x)
6617{
6618 struct slab *slab;
6619
6620 slab = virt_to_slab(x);
6621
6622 if (IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) ||
6623 kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
6624
6625 /*
6626 * Intentionally leak the object in these cases, because it
6627 * would be too dangerous to continue.
6628 */
6629 if (unlikely(!slab || (slab->slab_cache != s))) {
6630 warn_free_bad_obj(s, x);
6631 return;
6632 }
6633 }
6634
6635 trace_kmem_cache_free(_RET_IP_, x, s);
6636 slab_free(s, slab, x, _RET_IP_);
6637}
6638EXPORT_SYMBOL(kmem_cache_free);
6639
6640static inline size_t slab_ksize(struct slab *slab)
6641{
6642 struct kmem_cache *s = slab->slab_cache;
6643
6644#ifdef CONFIG_SLUB_DEBUG
6645 /*
6646 * Debugging requires use of the padding between object
6647 * and whatever may come after it.
6648 */
6649 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
6650 return s->object_size;
6651#endif
6652 if (s->flags & SLAB_KASAN)
6653 return s->object_size;
6654 /*
6655 * If we have the need to store the freelist pointer
6656 * or any other metadata back there then we can
6657 * only use the space before that information.
6658 */
6659 if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
6660 return s->inuse;
6661 else if (obj_exts_in_object(slab))
6662 return s->inuse;
6663 /*
6664 * Else we can use all the padding etc for the allocation
6665 */
6666 return s->size;
6667}
6668
6669static size_t __ksize(const void *object)
6670{
6671 struct page *page;
6672 struct slab *slab;
6673
6674 if (unlikely(object == ZERO_SIZE_PTR))
6675 return 0;
6676
6677 page = virt_to_page(object);
6678
6679 if (unlikely(PageLargeKmalloc(page)))
6680 return large_kmalloc_size(page);
6681
6682 slab = page_slab(page);
6683 /* Delete this after we're sure there are no users */
6684 if (WARN_ON(!slab))
6685 return page_size(page);
6686
6687#ifdef CONFIG_SLUB_DEBUG
6688 skip_orig_size_check(slab->slab_cache, object);
6689#endif
6690
6691 return slab_ksize(slab);
6692}
6693
6694/**
6695 * ksize -- Report full size of underlying allocation
6696 * @objp: pointer to the object
6697 *
6698 * This should only be used internally to query the true size of allocations.
6699 * It is not meant to be a way to discover the usable size of an allocation
6700 * after the fact. Instead, use kmalloc_size_roundup(). Using memory beyond
6701 * the originally requested allocation size may trigger KASAN, UBSAN_BOUNDS,
6702 * and/or FORTIFY_SOURCE.
6703 *
6704 * Return: size of the actual memory used by @objp in bytes
6705 */
6706size_t ksize(const void *objp)
6707{
6708 /*
6709 * We need to first check that the pointer to the object is valid.
6710 * The KASAN report printed from ksize() is more useful, then when
6711 * it's printed later when the behaviour could be undefined due to
6712 * a potential use-after-free or double-free.
6713 *
6714 * We use kasan_check_byte(), which is supported for the hardware
6715 * tag-based KASAN mode, unlike kasan_check_read/write().
6716 *
6717 * If the pointed to memory is invalid, we return 0 to avoid users of
6718 * ksize() writing to and potentially corrupting the memory region.
6719 *
6720 * We want to perform the check before __ksize(), to avoid potentially
6721 * crashing in __ksize() due to accessing invalid metadata.
6722 */
6723 if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp))
6724 return 0;
6725
6726 return kfence_ksize(objp) ?: __ksize(objp);
6727}
6728EXPORT_SYMBOL(ksize);
6729
6730static void free_large_kmalloc(struct page *page, void *object)
6731{
6732 unsigned int order = compound_order(page);
6733
6734 if (WARN_ON_ONCE(!PageLargeKmalloc(page))) {
6735 dump_page(page, "Not a kmalloc allocation");
6736 return;
6737 }
6738
6739 if (WARN_ON_ONCE(order == 0))
6740 pr_warn_once("object pointer: 0x%p\n", object);
6741
6742 kmemleak_free(object);
6743 kasan_kfree_large(object);
6744 kmsan_kfree_large(object);
6745
6746 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
6747 -(PAGE_SIZE << order));
6748 __ClearPageLargeKmalloc(page);
6749 free_frozen_pages(page, order);
6750}
6751
6752/*
6753 * Given an rcu_head embedded within an object obtained from kvmalloc at an
6754 * offset < 4k, free the object in question.
6755 */
6756void kvfree_rcu_cb(struct rcu_head *head)
6757{
6758 void *obj;
6759
6760 obj = kvmalloc_obj_start_addr(head);
6761
6762 if (is_vmalloc_addr(obj)) {
6763 vfree(obj);
6764 } else {
6765 struct page *page = virt_to_page(obj);
6766 struct slab *slab = page_slab(page);
6767
6768 if (slab)
6769 slab_free(slab->slab_cache, slab, obj, _RET_IP_);
6770 else
6771 free_large_kmalloc(page, obj);
6772 }
6773}
6774
6775/**
6776 * kfree - free previously allocated memory
6777 * @object: pointer returned by kmalloc(), kmalloc_nolock(), or kmem_cache_alloc()
6778 *
6779 * If @object is NULL, no operation is performed.
6780 */
6781void kfree(const void *object)
6782{
6783 struct page *page;
6784 struct slab *slab;
6785 struct kmem_cache *s;
6786 void *x = (void *)object;
6787
6788 trace_kfree(_RET_IP_, object);
6789
6790 if (unlikely(ZERO_OR_NULL_PTR(object)))
6791 return;
6792
6793 page = virt_to_page(object);
6794 slab = page_slab(page);
6795 if (!slab) {
6796 /* kmalloc_nolock() doesn't support large kmalloc */
6797 free_large_kmalloc(page, (void *)object);
6798 return;
6799 }
6800
6801 s = slab->slab_cache;
6802 slab_free(s, slab, x, _RET_IP_);
6803}
6804EXPORT_SYMBOL(kfree);
6805
6806/*
6807 * Can be called while holding raw_spinlock_t or from IRQ and NMI,
6808 * but ONLY for objects allocated by kmalloc_nolock().
6809 * Debug checks (like kmemleak and kfence) were skipped on allocation,
6810 * hence
6811 * obj = kmalloc(); kfree_nolock(obj);
6812 * will miss kmemleak/kfence book keeping and will cause false positives.
6813 * large_kmalloc is not supported either.
6814 */
6815void kfree_nolock(const void *object)
6816{
6817 struct slab *slab;
6818 struct kmem_cache *s;
6819 void *x = (void *)object;
6820
6821 if (unlikely(ZERO_OR_NULL_PTR(object)))
6822 return;
6823
6824 slab = virt_to_slab(object);
6825 if (unlikely(!slab)) {
6826 WARN_ONCE(1, "large_kmalloc is not supported by kfree_nolock()");
6827 return;
6828 }
6829
6830 s = slab->slab_cache;
6831
6832 memcg_slab_free_hook(s, slab, &x, 1);
6833 alloc_tagging_slab_free_hook(s, slab, &x, 1);
6834 /*
6835 * Unlike slab_free() do NOT call the following:
6836 * kmemleak_free_recursive(x, s->flags);
6837 * debug_check_no_locks_freed(x, s->object_size);
6838 * debug_check_no_obj_freed(x, s->object_size);
6839 * __kcsan_check_access(x, s->object_size, ..);
6840 * kfence_free(x);
6841 * since they take spinlocks or not safe from any context.
6842 */
6843 kmsan_slab_free(s, x);
6844 /*
6845 * If KASAN finds a kernel bug it will do kasan_report_invalid_free()
6846 * which will call raw_spin_lock_irqsave() which is technically
6847 * unsafe from NMI, but take chance and report kernel bug.
6848 * The sequence of
6849 * kasan_report_invalid_free() -> raw_spin_lock_irqsave() -> NMI
6850 * -> kfree_nolock() -> kasan_report_invalid_free() on the same CPU
6851 * is double buggy and deserves to deadlock.
6852 */
6853 if (kasan_slab_pre_free(s, x))
6854 return;
6855 /*
6856 * memcg, kasan_slab_pre_free are done for 'x'.
6857 * The only thing left is kasan_poison without quarantine,
6858 * since kasan quarantine takes locks and not supported from NMI.
6859 */
6860 kasan_slab_free(s, x, false, false, /* skip quarantine */true);
6861
6862 if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false)))
6863 return;
6864
6865 /*
6866 * __slab_free() can locklessly cmpxchg16 into a slab, but then it might
6867 * need to take spin_lock for further processing.
6868 * Avoid the complexity and simply add to a deferred list.
6869 */
6870 defer_free(s, x);
6871}
6872EXPORT_SYMBOL_GPL(kfree_nolock);
6873
6874static __always_inline __realloc_size(2) void *
6875__do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid, kmalloc_token_t token)
6876{
6877 void *ret;
6878 size_t ks = 0;
6879 int orig_size = 0;
6880 struct kmem_cache *s = NULL;
6881
6882 if (unlikely(ZERO_OR_NULL_PTR(p)))
6883 goto alloc_new;
6884
6885 /* Check for double-free. */
6886 if (!kasan_check_byte(p))
6887 return NULL;
6888
6889 if (is_kfence_address(p)) {
6890 ks = orig_size = kfence_ksize(p);
6891 } else {
6892 struct page *page = virt_to_page(p);
6893 struct slab *slab = page_slab(page);
6894
6895 if (!slab) {
6896 /* Big kmalloc object */
6897 ks = page_size(page);
6898 WARN_ON(ks <= KMALLOC_MAX_CACHE_SIZE);
6899 WARN_ON(p != page_address(page));
6900 } else {
6901 s = slab->slab_cache;
6902 orig_size = get_orig_size(s, (void *)p);
6903 ks = s->object_size;
6904 }
6905 }
6906
6907 /*
6908 * If reallocation is not necessary (e. g. the new size is less
6909 * than the current allocated size), the current allocation will be
6910 * preserved unless __GFP_THISNODE is set. In the latter case a new
6911 * allocation on the requested node will be attempted.
6912 */
6913 if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE &&
6914 nid != page_to_nid(virt_to_page(p)))
6915 goto alloc_new;
6916
6917 /* If the old object doesn't fit, allocate a bigger one */
6918 if (new_size > ks)
6919 goto alloc_new;
6920
6921 /* If the old object doesn't satisfy the new alignment, allocate a new one */
6922 if (!IS_ALIGNED((unsigned long)p, align))
6923 goto alloc_new;
6924
6925 /* Zero out spare memory. */
6926 if (want_init_on_alloc(flags)) {
6927 kasan_disable_current();
6928 if (orig_size && orig_size < new_size)
6929 memset(kasan_reset_tag(p) + orig_size, 0, new_size - orig_size);
6930 else
6931 memset(kasan_reset_tag(p) + new_size, 0, ks - new_size);
6932 kasan_enable_current();
6933 }
6934
6935 /* Setup kmalloc redzone when needed */
6936 if (s && slub_debug_orig_size(s)) {
6937 set_orig_size(s, (void *)p, new_size);
6938 if (s->flags & SLAB_RED_ZONE && new_size < ks)
6939 memset_no_sanitize_memory(kasan_reset_tag(p) + new_size,
6940 SLUB_RED_ACTIVE, ks - new_size);
6941 }
6942
6943 p = kasan_krealloc(p, new_size, flags);
6944 return (void *)p;
6945
6946alloc_new:
6947 ret = __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(new_size, NULL, token), flags, nid, _RET_IP_);
6948 if (ret && p) {
6949 /* Disable KASAN checks as the object's redzone is accessed. */
6950 kasan_disable_current();
6951 memcpy(ret, kasan_reset_tag(p), min(new_size, (size_t)(orig_size ?: ks)));
6952 kasan_enable_current();
6953 }
6954
6955 return ret;
6956}
6957
6958void *krealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(new_size, token), unsigned long align,
6959 gfp_t flags, int nid)
6960{
6961 void *ret;
6962
6963 if (unlikely(!new_size)) {
6964 kfree(p);
6965 return ZERO_SIZE_PTR;
6966 }
6967
6968 ret = __do_krealloc(p, new_size, align, flags, nid, PASS_TOKEN_PARAM(token));
6969 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
6970 kfree(p);
6971
6972 return ret;
6973}
6974EXPORT_SYMBOL(krealloc_node_align_noprof);
6975
6976static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
6977{
6978 /*
6979 * We want to attempt a large physically contiguous block first because
6980 * it is less likely to fragment multiple larger blocks and therefore
6981 * contribute to a long term fragmentation less than vmalloc fallback.
6982 * However make sure that larger requests are not too disruptive - i.e.
6983 * do not direct reclaim unless physically continuous memory is preferred
6984 * (__GFP_RETRY_MAYFAIL mode). We still kick in kswapd/kcompactd to
6985 * start working in the background
6986 */
6987 if (size > PAGE_SIZE) {
6988 flags |= __GFP_NOWARN;
6989
6990 if (!(flags & __GFP_RETRY_MAYFAIL))
6991 flags &= ~__GFP_DIRECT_RECLAIM;
6992
6993 /* nofail semantic is implemented by the vmalloc fallback */
6994 flags &= ~__GFP_NOFAIL;
6995 }
6996
6997 return flags;
6998}
6999
7000void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align,
7001 gfp_t flags, int node)
7002{
7003 bool allow_block;
7004 void *ret;
7005 const struct slab_alloc_context ac = {
7006 .caller_addr = _RET_IP_,
7007 .orig_size = size,
7008 .alloc_flags = SLAB_ALLOC_DEFAULT,
7009 };
7010
7011 /*
7012 * It doesn't really make sense to fallback to vmalloc for sub page
7013 * requests
7014 */
7015 ret = __do_kmalloc_node(PASS_BUCKET_PARAM(b),
7016 kmalloc_gfp_adjust(flags, size),
7017 node, PASS_TOKEN_PARAM(token), &ac);
7018 if (ret || size <= PAGE_SIZE)
7019 return ret;
7020
7021 /* Don't even allow crazy sizes */
7022 if (unlikely(size > INT_MAX)) {
7023 WARN_ON_ONCE(!(flags & __GFP_NOWARN));
7024 return NULL;
7025 }
7026
7027 /*
7028 * For non-blocking the VM_ALLOW_HUGE_VMAP is not used
7029 * because the huge-mapping path in vmalloc contains at
7030 * least one might_sleep() call.
7031 *
7032 * TODO: Revise huge-mapping path to support non-blocking
7033 * flags.
7034 */
7035 allow_block = gfpflags_allow_blocking(flags);
7036
7037 /*
7038 * kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
7039 * since the callers already cannot assume anything
7040 * about the resulting pointer, and cannot play
7041 * protection games.
7042 */
7043 return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END,
7044 flags, PAGE_KERNEL, allow_block ? VM_ALLOW_HUGE_VMAP:0,
7045 node, __builtin_return_address(0));
7046}
7047EXPORT_SYMBOL(__kvmalloc_node_noprof);
7048
7049/**
7050 * kvfree() - Free memory.
7051 * @addr: Pointer to allocated memory.
7052 *
7053 * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
7054 * It is slightly more efficient to use kfree() or vfree() if you are certain
7055 * that you know which one to use.
7056 *
7057 * Context: Either preemptible task context or not-NMI interrupt.
7058 */
7059void kvfree(const void *addr)
7060{
7061 if (is_vmalloc_addr(addr))
7062 vfree(addr);
7063 else
7064 kfree(addr);
7065}
7066EXPORT_SYMBOL(kvfree);
7067
7068/**
7069 * kvfree_atomic() - Free memory.
7070 * @addr: Pointer to allocated memory.
7071 *
7072 * Same as kvfree(), but uses vfree_atomic() for vmalloc
7073 * backed memory. Must not be called from NMI context.
7074 */
7075void kvfree_atomic(const void *addr)
7076{
7077 if (is_vmalloc_addr(addr))
7078 vfree_atomic(addr);
7079 else
7080 kfree(addr);
7081}
7082EXPORT_SYMBOL(kvfree_atomic);
7083
7084/**
7085 * kvfree_sensitive - Free a data object containing sensitive information.
7086 * @addr: address of the data object to be freed.
7087 * @len: length of the data object.
7088 *
7089 * Use the special memzero_explicit() function to clear the content of a
7090 * kvmalloc'ed object containing sensitive data to make sure that the
7091 * compiler won't optimize out the data clearing.
7092 */
7093void kvfree_sensitive(const void *addr, size_t len)
7094{
7095 if (likely(!ZERO_OR_NULL_PTR(addr))) {
7096 memzero_explicit((void *)addr, len);
7097 kvfree(addr);
7098 }
7099}
7100EXPORT_SYMBOL(kvfree_sensitive);
7101
7102void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align,
7103 gfp_t flags, int nid)
7104{
7105 void *n;
7106
7107 if (is_vmalloc_addr(p))
7108 return vrealloc_node_align_noprof(p, size, align, flags, nid);
7109
7110 n = krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, token), align, kmalloc_gfp_adjust(flags, size), nid);
7111 if (!n) {
7112 /* We failed to krealloc(), fall back to kvmalloc(). */
7113 n = __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), align, flags, nid);
7114 if (!n)
7115 return NULL;
7116
7117 if (p) {
7118 /* We already know that `p` is not a vmalloc address. */
7119 kasan_disable_current();
7120 memcpy(n, kasan_reset_tag(p), min(size, ksize(p)));
7121 kasan_enable_current();
7122
7123 kfree(p);
7124 }
7125 }
7126
7127 return n;
7128}
7129EXPORT_SYMBOL(kvrealloc_node_align_noprof);
7130
7131struct detached_freelist {
7132 struct slab *slab;
7133 void *tail;
7134 void *freelist;
7135 int cnt;
7136 struct kmem_cache *s;
7137};
7138
7139/*
7140 * This function progressively scans the array with free objects (with
7141 * a limited look ahead) and extract objects belonging to the same
7142 * slab. It builds a detached freelist directly within the given
7143 * slab/objects. This can happen without any need for
7144 * synchronization, because the objects are owned by running process.
7145 * The freelist is build up as a single linked list in the objects.
7146 * The idea is, that this detached freelist can then be bulk
7147 * transferred to the real freelist(s), but only requiring a single
7148 * synchronization primitive. Look ahead in the array is limited due
7149 * to performance reasons.
7150 */
7151static inline
7152int build_detached_freelist(struct kmem_cache *s, size_t size,
7153 void **p, struct detached_freelist *df)
7154{
7155 int lookahead = 3;
7156 void *object;
7157 struct page *page;
7158 struct slab *slab;
7159 size_t same;
7160
7161 object = p[--size];
7162 page = virt_to_page(object);
7163 slab = page_slab(page);
7164 if (!s) {
7165 /* Handle kalloc'ed objects */
7166 if (!slab) {
7167 free_large_kmalloc(page, object);
7168 df->slab = NULL;
7169 return size;
7170 }
7171 /* Derive kmem_cache from object */
7172 df->slab = slab;
7173 df->s = slab->slab_cache;
7174 } else {
7175 df->slab = slab;
7176 df->s = s;
7177 }
7178
7179 /* Start new detached freelist */
7180 df->tail = object;
7181 df->freelist = object;
7182 df->cnt = 1;
7183
7184 if (is_kfence_address(object))
7185 return size;
7186
7187 set_freepointer(df->s, object, NULL);
7188
7189 same = size;
7190 while (size) {
7191 object = p[--size];
7192 /* df->slab is always set at this point */
7193 if (df->slab == virt_to_slab(object)) {
7194 /* Opportunity build freelist */
7195 set_freepointer(df->s, object, df->freelist);
7196 df->freelist = object;
7197 df->cnt++;
7198 same--;
7199 if (size != same)
7200 swap(p[size], p[same]);
7201 continue;
7202 }
7203
7204 /* Limit look ahead search */
7205 if (!--lookahead)
7206 break;
7207 }
7208
7209 return same;
7210}
7211
7212/*
7213 * Internal bulk free of objects that were not initialised by the post alloc
7214 * hooks and thus should not be processed by the free hooks
7215 */
7216static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
7217{
7218 if (!size)
7219 return;
7220
7221 do {
7222 struct detached_freelist df;
7223
7224 size = build_detached_freelist(s, size, p, &df);
7225 if (!df.slab)
7226 continue;
7227
7228 if (kfence_free(df.freelist))
7229 continue;
7230
7231 __slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
7232 _RET_IP_);
7233 } while (likely(size));
7234}
7235
7236/* Note that interrupts must be enabled when calling this function. */
7237void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
7238{
7239 if (!size)
7240 return;
7241
7242 /*
7243 * freeing to sheaves is so incompatible with the detached freelist so
7244 * once we go that way, we have to do everything differently
7245 */
7246 if (s && cache_has_sheaves(s)) {
7247 free_to_pcs_bulk(s, size, p);
7248 return;
7249 }
7250
7251 do {
7252 struct detached_freelist df;
7253
7254 size = build_detached_freelist(s, size, p, &df);
7255 if (!df.slab)
7256 continue;
7257
7258 slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size],
7259 df.cnt, _RET_IP_);
7260 } while (likely(size));
7261}
7262EXPORT_SYMBOL(kmem_cache_free_bulk);
7263
7264static unsigned int
7265__refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
7266 unsigned int max, struct kmem_cache_node *n,
7267 bool allow_spin)
7268{
7269 struct partial_bulk_context pc;
7270 struct slab *slab, *slab2;
7271 unsigned int refilled = 0;
7272 unsigned long flags;
7273 void *object;
7274
7275 pc.flags = gfp;
7276 pc.min_objects = min;
7277 pc.max_objects = max;
7278
7279 if (!get_partial_node_bulk(s, n, &pc, allow_spin))
7280 return 0;
7281
7282 list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) {
7283
7284 unsigned int count;
7285
7286 list_del(&slab->slab_list);
7287
7288 object = get_freelist_nofreeze(s, slab, &count);
7289
7290 while (count && refilled < max) {
7291 p[refilled] = object;
7292 object = get_freepointer(s, object);
7293 maybe_wipe_obj_freeptr(s, p[refilled]);
7294
7295 refilled++;
7296 count--;
7297 }
7298
7299 /*
7300 * Freelist had more objects than we can accommodate, we need to
7301 * free them back. First we try to be optimistic and assume the
7302 * slab is still full since we just detached its freelist.
7303 * Otherwise we must find the tail object.
7304 */
7305 if (unlikely(count)) {
7306 void *head = object;
7307 void *tail;
7308
7309 if (__slab_try_return_freelist(s, n, slab, head, count))
7310 break;
7311
7312 do {
7313 tail = object;
7314 object = get_freepointer(s, object);
7315 } while (object);
7316 __slab_free(s, slab, head, tail, count, _RET_IP_);
7317 }
7318
7319 if (refilled >= max)
7320 break;
7321 }
7322
7323 if (unlikely(!list_empty(&pc.slabs))) {
7324 spin_lock_irqsave(&n->list_lock, flags);
7325
7326 list_for_each_entry(slab, &pc.slabs, slab_list)
7327 set_node_partial_state(n, slab);
7328
7329 list_splice_tail(&pc.slabs, &n->partial);
7330
7331 spin_unlock_irqrestore(&n->list_lock, flags);
7332 }
7333
7334 return refilled;
7335}
7336
7337#ifdef CONFIG_NUMA
7338static unsigned int
7339__refill_objects_any(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
7340 unsigned int max)
7341{
7342 struct zonelist *zonelist;
7343 struct zoneref *z;
7344 struct zone *zone;
7345 enum zone_type highest_zoneidx = gfp_zone(gfp);
7346 unsigned int cpuset_mems_cookie;
7347 unsigned int refilled = 0;
7348
7349 /* see get_from_any_partial() for the defrag ratio description */
7350 if (!s->remote_node_defrag_ratio ||
7351 get_cycles() % 1024 > s->remote_node_defrag_ratio)
7352 return 0;
7353
7354 do {
7355 cpuset_mems_cookie = read_mems_allowed_begin();
7356 zonelist = node_zonelist(mempolicy_slab_node(), gfp);
7357 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
7358 struct kmem_cache_node *n;
7359 unsigned int r;
7360
7361 n = get_node(s, zone_to_nid(zone));
7362
7363 if (!n || !cpuset_zone_allowed(zone, gfp) ||
7364 n->nr_partial <= s->min_partial)
7365 continue;
7366
7367 r = __refill_objects_node(s, p, gfp, min, max, n,
7368 /* allow_spin = */ false);
7369 refilled += r;
7370
7371 if (r >= min) {
7372 /*
7373 * Don't check read_mems_allowed_retry() here -
7374 * if mems_allowed was updated in parallel, that
7375 * was a harmless race between allocation and
7376 * the cpuset update
7377 */
7378 return refilled;
7379 }
7380 p += r;
7381 min -= r;
7382 max -= r;
7383 }
7384 } while (read_mems_allowed_retry(cpuset_mems_cookie));
7385
7386 return refilled;
7387}
7388#else
7389static inline unsigned int
7390__refill_objects_any(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
7391 unsigned int max)
7392{
7393 return 0;
7394}
7395#endif
7396
7397static unsigned int
7398refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
7399 unsigned int max)
7400{
7401 int local_node = numa_mem_id();
7402 unsigned int refilled;
7403 struct slab *slab;
7404
7405 refilled = __refill_objects_node(s, p, gfp, min, max,
7406 get_node(s, local_node),
7407 /* allow_spin = */ true);
7408 if (refilled >= min)
7409 return refilled;
7410
7411 refilled += __refill_objects_any(s, p + refilled, gfp, min - refilled,
7412 max - refilled);
7413 if (refilled >= min)
7414 return refilled;
7415
7416new_slab:
7417
7418 slab = new_slab(s, gfp, SLAB_ALLOC_DEFAULT, local_node);
7419 if (!slab)
7420 goto out;
7421
7422 stat(s, ALLOC_SLAB);
7423
7424 refilled += alloc_from_new_slab(s, slab, p + refilled, max - refilled,
7425 /* allow_spin = */ true);
7426
7427 if (refilled < min)
7428 goto new_slab;
7429
7430out:
7431 return refilled;
7432}
7433
7434static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
7435 size_t size, void **p)
7436{
7437 int i;
7438
7439 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
7440 const struct slab_alloc_context ac = {
7441 .caller_addr = _RET_IP_,
7442 .orig_size = s->object_size,
7443 .alloc_flags = SLAB_ALLOC_DEFAULT,
7444 };
7445 for (i = 0; i < size; i++) {
7446
7447 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, &ac);
7448 if (unlikely(!p[i]))
7449 goto error;
7450
7451 maybe_wipe_obj_freeptr(s, p[i]);
7452 }
7453 } else {
7454 i = refill_objects(s, p, flags, size, size);
7455 if (i < size)
7456 goto error;
7457 stat_add(s, ALLOC_SLOWPATH, i);
7458 }
7459
7460 return true;
7461
7462error:
7463 __kmem_cache_free_bulk(s, i, p);
7464 return false;
7465}
7466
7467/**
7468 * kmem_cache_alloc_bulk - Allocate multiple objects
7469 * @s: The cache to allocate from
7470 * @flags: GFP_* flags. See kmalloc().
7471 * @size: Number of objects to allocate
7472 * @p: Array of allocated objects
7473 *
7474 * Allocate @size objects from @s and places them into @p. @size must be larger
7475 * than 0.
7476 *
7477 * Interrupts must be enabled when calling this function.
7478 *
7479 * Unlike alloc_pages_bulk(), this function does not check for already allocated
7480 * objects in @p, and thus the caller does not need to zero it.
7481 *
7482 * Return: %true if the allocation succeeded, or %false if it failed.
7483 */
7484bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
7485 size_t size, void **p)
7486{
7487 unsigned int i = 0;
7488 void *kfence_obj;
7489 const struct slab_alloc_context ac = {
7490 .orig_size = s->object_size,
7491 .alloc_flags = SLAB_ALLOC_DEFAULT,
7492 };
7493
7494 if (!size)
7495 return false;
7496
7497 s = slab_pre_alloc_hook(s, flags);
7498 if (unlikely(!s))
7499 return false;
7500
7501 /*
7502 * to make things simpler, only assume at most once kfence allocated
7503 * object per bulk allocation and choose its index randomly
7504 */
7505 kfence_obj = kfence_alloc(s, s->object_size, flags);
7506
7507 if (unlikely(kfence_obj)) {
7508 if (unlikely(size == 1)) {
7509 p[0] = kfence_obj;
7510 goto out;
7511 }
7512 size--;
7513 }
7514
7515 i = alloc_from_pcs_bulk(s, size, p);
7516 if (i < size) {
7517 /*
7518 * If we ran out of memory, don't bother with freeing back to
7519 * the percpu sheaves, we have bigger problems.
7520 */
7521 if (unlikely(!__kmem_cache_alloc_bulk(s, flags, size - i,
7522 p + i))) {
7523 if (i > 0)
7524 __kmem_cache_free_bulk(s, i, p);
7525 if (kfence_obj)
7526 __kfence_free(kfence_obj);
7527 return false;
7528 }
7529 }
7530
7531 if (unlikely(kfence_obj)) {
7532 int idx = get_random_u32_below(size + 1);
7533
7534 if (idx != size)
7535 p[size] = p[idx];
7536 p[idx] = kfence_obj;
7537
7538 size++;
7539 }
7540
7541out:
7542 /* memcg and kmem_cache debug support and memory initialization */
7543 return likely(slab_post_alloc_hook(s, flags, size, p, &ac));
7544}
7545EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);
7546
7547/*
7548 * Object placement in a slab is made very easy because we always start at
7549 * offset 0. If we tune the size of the object to the alignment then we can
7550 * get the required alignment by putting one properly sized object after
7551 * another.
7552 *
7553 * Notice that the allocation order determines the sizes of the per cpu
7554 * caches. Each processor has always one slab available for allocations.
7555 * Increasing the allocation order reduces the number of times that slabs
7556 * must be moved on and off the partial lists and is therefore a factor in
7557 * locking overhead.
7558 */
7559
7560/*
7561 * Minimum / Maximum order of slab pages. This influences locking overhead
7562 * and slab fragmentation. A higher order reduces the number of partial slabs
7563 * and increases the number of allocations possible without having to
7564 * take the list_lock.
7565 */
7566static unsigned int slub_min_order;
7567static unsigned int slub_max_order =
7568 IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER;
7569static unsigned int slub_min_objects;
7570
7571/*
7572 * Calculate the order of allocation given an slab object size.
7573 *
7574 * The order of allocation has significant impact on performance and other
7575 * system components. Generally order 0 allocations should be preferred since
7576 * order 0 does not cause fragmentation in the page allocator. Larger objects
7577 * be problematic to put into order 0 slabs because there may be too much
7578 * unused space left. We go to a higher order if more than 1/16th of the slab
7579 * would be wasted.
7580 *
7581 * In order to reach satisfactory performance we must ensure that a minimum
7582 * number of objects is in one slab. Otherwise we may generate too much
7583 * activity on the partial lists which requires taking the list_lock. This is
7584 * less a concern for large slabs though which are rarely used.
7585 *
7586 * slab_max_order specifies the order where we begin to stop considering the
7587 * number of objects in a slab as critical. If we reach slab_max_order then
7588 * we try to keep the page order as low as possible. So we accept more waste
7589 * of space in favor of a small page order.
7590 *
7591 * Higher order allocations also allow the placement of more objects in a
7592 * slab and thereby reduce object handling overhead. If the user has
7593 * requested a higher minimum order then we start with that one instead of
7594 * the smallest order which will fit the object.
7595 */
7596static inline unsigned int calc_slab_order(unsigned int size,
7597 unsigned int min_order, unsigned int max_order,
7598 unsigned int fract_leftover)
7599{
7600 unsigned int order;
7601
7602 for (order = min_order; order <= max_order; order++) {
7603
7604 unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
7605 unsigned int rem;
7606
7607 rem = slab_size % size;
7608
7609 if (rem <= slab_size / fract_leftover)
7610 break;
7611 }
7612
7613 return order;
7614}
7615
7616static inline int calculate_order(unsigned int size)
7617{
7618 unsigned int order;
7619 unsigned int min_objects;
7620 unsigned int max_objects;
7621 unsigned int min_order;
7622
7623 min_objects = slub_min_objects;
7624 if (!min_objects) {
7625 /*
7626 * Some architectures will only update present cpus when
7627 * onlining them, so don't trust the number if it's just 1. But
7628 * we also don't want to use nr_cpu_ids always, as on some other
7629 * architectures, there can be many possible cpus, but never
7630 * onlined. Here we compromise between trying to avoid too high
7631 * order on systems that appear larger than they are, and too
7632 * low order on systems that appear smaller than they are.
7633 */
7634 unsigned int nr_cpus = num_present_cpus();
7635 if (nr_cpus <= 1)
7636 nr_cpus = nr_cpu_ids;
7637 min_objects = 4 * (fls(nr_cpus) + 1);
7638 }
7639 /* min_objects can't be 0 because get_order(0) is undefined */
7640 max_objects = max(order_objects(slub_max_order, size), 1U);
7641 min_objects = min(min_objects, max_objects);
7642
7643 min_order = max_t(unsigned int, slub_min_order,
7644 get_order(min_objects * size));
7645 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
7646 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
7647
7648 /*
7649 * Attempt to find best configuration for a slab. This works by first
7650 * attempting to generate a layout with the best possible configuration
7651 * and backing off gradually.
7652 *
7653 * We start with accepting at most 1/16 waste and try to find the
7654 * smallest order from min_objects-derived/slab_min_order up to
7655 * slab_max_order that will satisfy the constraint. Note that increasing
7656 * the order can only result in same or less fractional waste, not more.
7657 *
7658 * If that fails, we increase the acceptable fraction of waste and try
7659 * again. The last iteration with fraction of 1/2 would effectively
7660 * accept any waste and give us the order determined by min_objects, as
7661 * long as at least single object fits within slab_max_order.
7662 */
7663 for (unsigned int fraction = 16; fraction > 1; fraction /= 2) {
7664 order = calc_slab_order(size, min_order, slub_max_order,
7665 fraction);
7666 if (order <= slub_max_order)
7667 return order;
7668 }
7669
7670 /*
7671 * Doh this slab cannot be placed using slab_max_order.
7672 */
7673 order = get_order(size);
7674 if (order <= MAX_PAGE_ORDER)
7675 return order;
7676 return -ENOSYS;
7677}
7678
7679static void
7680init_kmem_cache_node(struct kmem_cache_node *n)
7681{
7682 n->nr_partial = 0;
7683 spin_lock_init(&n->list_lock);
7684 INIT_LIST_HEAD(&n->partial);
7685#ifdef CONFIG_SLUB_DEBUG
7686 atomic_long_set(&n->nr_slabs, 0);
7687 atomic_long_set(&n->total_objects, 0);
7688 INIT_LIST_HEAD(&n->full);
7689#endif
7690}
7691
7692#ifdef CONFIG_SLUB_STATS
7693static inline int alloc_kmem_cache_stats(struct kmem_cache *s)
7694{
7695 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
7696 NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH *
7697 sizeof(struct kmem_cache_stats));
7698
7699 s->cpu_stats = alloc_percpu(struct kmem_cache_stats);
7700
7701 if (!s->cpu_stats)
7702 return 0;
7703
7704 return 1;
7705}
7706#endif
7707
7708static int init_percpu_sheaves(struct kmem_cache *s)
7709{
7710 static struct slab_sheaf bootstrap_sheaf = {};
7711 int cpu;
7712
7713 for_each_possible_cpu(cpu) {
7714 struct slub_percpu_sheaves *pcs;
7715
7716 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
7717
7718 local_trylock_init(&pcs->lock);
7719
7720 /*
7721 * Bootstrap sheaf has zero size so fast-path allocation fails.
7722 * It has also size == s->sheaf_capacity, so fast-path free
7723 * fails. In the slow paths we recognize the situation by
7724 * checking s->sheaf_capacity. This allows fast paths to assume
7725 * s->cpu_sheaves and pcs->main always exists and are valid.
7726 * It's also safe to share the single static bootstrap_sheaf
7727 * with zero-sized objects array as it's never modified.
7728 *
7729 * Bootstrap_sheaf also has NULL pointer to kmem_cache so we
7730 * recognize it and not attempt to free it when destroying the
7731 * cache.
7732 *
7733 * We keep bootstrap_sheaf for kmem_cache and kmem_cache_node,
7734 * caches with debug enabled, and all caches with SLUB_TINY.
7735 * For kmalloc caches it's used temporarily during the initial
7736 * bootstrap.
7737 */
7738 if (!s->sheaf_capacity)
7739 pcs->main = &bootstrap_sheaf;
7740 else
7741 pcs->main = alloc_empty_sheaf(s, GFP_KERNEL, SLAB_ALLOC_DEFAULT);
7742
7743 if (!pcs->main)
7744 return -ENOMEM;
7745 }
7746
7747 return 0;
7748}
7749
7750static struct kmem_cache *kmem_cache_node;
7751
7752/*
7753 * No kmalloc_node yet so do it by hand. We know that this is the first
7754 * slab on the node for this slabcache. There are no concurrent accesses
7755 * possible.
7756 *
7757 * Note that this function only works on the kmem_cache_node
7758 * when allocating for the kmem_cache_node. This is used for bootstrapping
7759 * memory on a fresh node that has no slab structures yet.
7760 */
7761static void early_kmem_cache_node_alloc(int node)
7762{
7763 struct slab *slab;
7764 struct kmem_cache_node *n;
7765 struct slab_obj_iter iter;
7766
7767 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
7768
7769 slab = new_slab(kmem_cache_node, GFP_NOWAIT, SLAB_ALLOC_DEFAULT, node);
7770
7771 BUG_ON(!slab);
7772 if (slab_nid(slab) != node) {
7773 pr_err("SLUB: Unable to allocate memory from node %d\n", node);
7774 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
7775 }
7776
7777 init_slab_obj_iter(kmem_cache_node, slab, &iter, true);
7778
7779 n = next_slab_obj(kmem_cache_node, &iter);
7780 BUG_ON(!n);
7781
7782 slab->inuse = 1;
7783 build_slab_freelist(kmem_cache_node, slab, &iter);
7784
7785#ifdef CONFIG_SLUB_DEBUG
7786 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
7787#endif
7788 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
7789 kmem_cache_node->per_node[node].node = n;
7790 init_kmem_cache_node(n);
7791 inc_slabs_node(kmem_cache_node, node, slab->objects);
7792
7793 /*
7794 * No locks need to be taken here as it has just been
7795 * initialized and there is no concurrent access.
7796 */
7797 __add_partial(n, slab, ADD_TO_HEAD);
7798}
7799
7800static void free_kmem_cache_nodes(struct kmem_cache *s)
7801{
7802 int node;
7803 struct kmem_cache_node *n;
7804
7805 for_each_node(node) {
7806 struct node_barn *barn = get_barn_node(s, node);
7807
7808 if (!barn)
7809 continue;
7810
7811 WARN_ON(barn->nr_full);
7812 WARN_ON(barn->nr_empty);
7813 kfree(barn);
7814 s->per_node[node].barn = NULL;
7815 }
7816
7817 for_each_kmem_cache_node(s, node, n) {
7818 s->per_node[node].node = NULL;
7819 kmem_cache_free(kmem_cache_node, n);
7820 }
7821}
7822
7823void __kmem_cache_release(struct kmem_cache *s)
7824{
7825 cache_random_seq_destroy(s);
7826 pcs_destroy(s);
7827#ifdef CONFIG_SLUB_STATS
7828 free_percpu(s->cpu_stats);
7829#endif
7830 free_kmem_cache_nodes(s);
7831}
7832
7833static int init_kmem_cache_nodes(struct kmem_cache *s)
7834{
7835 int node;
7836
7837 for_each_node_mask(node, slab_nodes) {
7838 struct kmem_cache_node *n;
7839
7840 if (slab_state == DOWN) {
7841 early_kmem_cache_node_alloc(node);
7842 continue;
7843 }
7844
7845 n = kmem_cache_alloc_node(kmem_cache_node,
7846 GFP_KERNEL, node);
7847 if (!n)
7848 return 0;
7849
7850 init_kmem_cache_node(n);
7851 s->per_node[node].node = n;
7852 }
7853
7854 if (slab_state == DOWN || !cache_has_sheaves(s))
7855 return 1;
7856
7857 for_each_node_mask(node, slab_barn_nodes) {
7858 struct node_barn *barn;
7859
7860 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
7861
7862 if (!barn)
7863 return 0;
7864
7865 barn_init(barn);
7866 s->per_node[node].barn = barn;
7867 }
7868
7869 return 1;
7870}
7871
7872static unsigned int calculate_sheaf_capacity(struct kmem_cache *s,
7873 struct kmem_cache_args *args)
7874
7875{
7876 unsigned int capacity;
7877 size_t size;
7878
7879
7880 if (IS_ENABLED(CONFIG_SLUB_TINY) || s->flags & SLAB_DEBUG_FLAGS)
7881 return 0;
7882
7883 /*
7884 * Bootstrap caches can't have sheaves for now (SLAB_NO_SHEAVES).
7885 * SLAB_NOLEAKTRACE caches (e.g., kmemleak's object_cache) must not
7886 * have sheaves to avoid recursion when sheaf allocation triggers
7887 * kmemleak tracking.
7888 */
7889 if (s->flags & (SLAB_NO_SHEAVES | SLAB_NOLEAKTRACE))
7890 return 0;
7891
7892 /*
7893 * For now we use roughly similar formula (divided by two as there are
7894 * two percpu sheaves) as what was used for percpu partial slabs, which
7895 * should result in similar lock contention (barn or list_lock)
7896 */
7897 if (s->size >= PAGE_SIZE)
7898 capacity = 4;
7899 else if (s->size >= 1024)
7900 capacity = 12;
7901 else if (s->size >= 256)
7902 capacity = 26;
7903 else
7904 capacity = 60;
7905
7906 /* Increment capacity to make sheaf exactly a kmalloc size bucket */
7907 size = struct_size_t(struct slab_sheaf, objects, capacity);
7908 size = kmalloc_size_roundup(size);
7909 capacity = (size - struct_size_t(struct slab_sheaf, objects, 0)) / sizeof(void *);
7910
7911 /*
7912 * Respect an explicit request for capacity that's typically motivated by
7913 * expected maximum size of kmem_cache_prefill_sheaf() to not end up
7914 * using low-performance oversize sheaves
7915 */
7916 return max(capacity, args->sheaf_capacity);
7917}
7918
7919/*
7920 * calculate_sizes() determines the order and the distribution of data within
7921 * a slab object.
7922 */
7923static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
7924{
7925 slab_flags_t flags = s->flags;
7926 unsigned int size = s->object_size;
7927 unsigned int aligned_size;
7928 unsigned int order;
7929
7930 /*
7931 * Round up object size to the next word boundary. We can only
7932 * place the free pointer at word boundaries and this determines
7933 * the possible location of the free pointer.
7934 */
7935 size = ALIGN(size, sizeof(void *));
7936
7937#ifdef CONFIG_SLUB_DEBUG
7938 /*
7939 * Determine if we can poison the object itself. If the user of
7940 * the slab may touch the object after free or before allocation
7941 * then we should never poison the object itself.
7942 */
7943 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
7944 !s->ctor)
7945 s->flags |= __OBJECT_POISON;
7946 else
7947 s->flags &= ~__OBJECT_POISON;
7948
7949
7950 /*
7951 * If we are Redzoning and there is no space between the end of the
7952 * object and the following fields, add one word so the right Redzone
7953 * is non-empty.
7954 */
7955 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
7956 size += sizeof(void *);
7957#endif
7958
7959 /*
7960 * With that we have determined the number of bytes in actual use
7961 * by the object and redzoning.
7962 */
7963 s->inuse = size;
7964
7965 if (((flags & SLAB_TYPESAFE_BY_RCU) && !args->use_freeptr_offset) ||
7966 (flags & SLAB_POISON) ||
7967 (s->ctor && !args->use_freeptr_offset) ||
7968 ((flags & SLAB_RED_ZONE) &&
7969 (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) {
7970 /*
7971 * Relocate free pointer after the object if it is not
7972 * permitted to overwrite the first word of the object on
7973 * kmem_cache_free.
7974 *
7975 * This is the case if we do RCU, have a constructor, are
7976 * poisoning the objects, or are redzoning an object smaller
7977 * than sizeof(void *) or are redzoning an object with
7978 * slub_debug_orig_size() enabled, in which case the right
7979 * redzone may be extended.
7980 *
7981 * The assumption that s->offset >= s->inuse means free
7982 * pointer is outside of the object is used in the
7983 * freeptr_outside_object() function. If that is no
7984 * longer true, the function needs to be modified.
7985 */
7986 s->offset = size;
7987 size += sizeof(void *);
7988 } else if (((flags & SLAB_TYPESAFE_BY_RCU) || s->ctor) &&
7989 args->use_freeptr_offset) {
7990 s->offset = args->freeptr_offset;
7991 } else {
7992 /*
7993 * Store freelist pointer near middle of object to keep
7994 * it away from the edges of the object to avoid small
7995 * sized over/underflows from neighboring allocations.
7996 */
7997 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
7998 }
7999
8000#ifdef CONFIG_SLUB_DEBUG
8001 if (flags & SLAB_STORE_USER) {
8002 /*
8003 * Need to store information about allocs and frees after
8004 * the object.
8005 */
8006 size += 2 * sizeof(struct track);
8007
8008 /* Save the original kmalloc request size */
8009 if (flags & SLAB_KMALLOC)
8010 size += sizeof(unsigned long);
8011 }
8012#endif
8013
8014 kasan_cache_create(s, &size, &s->flags);
8015#ifdef CONFIG_SLUB_DEBUG
8016 if (flags & SLAB_RED_ZONE) {
8017 /*
8018 * Add some empty padding so that we can catch
8019 * overwrites from earlier objects rather than let
8020 * tracking information or the free pointer be
8021 * corrupted if a user writes before the start
8022 * of the object.
8023 */
8024 size += sizeof(void *);
8025
8026 s->red_left_pad = sizeof(void *);
8027 s->red_left_pad = ALIGN(s->red_left_pad, s->align);
8028 size += s->red_left_pad;
8029 }
8030#endif
8031
8032 /*
8033 * SLUB stores one object immediately after another beginning from
8034 * offset 0. In order to align the objects we have to simply size
8035 * each object to conform to the alignment.
8036 */
8037 aligned_size = ALIGN(size, s->align);
8038#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
8039 if (slab_args_unmergeable(args, s->flags) &&
8040 (aligned_size - size >= cache_obj_ext_size(s)))
8041 s->flags |= SLAB_OBJ_EXT_IN_OBJ;
8042#endif
8043 size = aligned_size;
8044
8045 s->size = size;
8046 s->reciprocal_size = reciprocal_value(size);
8047 order = calculate_order(size);
8048
8049 if ((int)order < 0)
8050 return 0;
8051
8052 s->allocflags = __GFP_COMP;
8053
8054 if (s->flags & SLAB_CACHE_DMA)
8055 s->allocflags |= GFP_DMA;
8056
8057 if (s->flags & SLAB_CACHE_DMA32)
8058 s->allocflags |= GFP_DMA32;
8059
8060 if (s->flags & SLAB_RECLAIM_ACCOUNT)
8061 s->allocflags |= __GFP_RECLAIMABLE;
8062
8063 /*
8064 * For kmalloc caches we enable sheaves later by
8065 * bootstrap_kmalloc_sheaves() to avoid recursion.
8066 */
8067 if (!is_kmalloc_cache(s))
8068 s->sheaf_capacity = calculate_sheaf_capacity(s, args);
8069
8070 /*
8071 * Determine the number of objects per slab
8072 */
8073 s->oo = oo_make(order, size);
8074 s->min = oo_make(get_order(size), size);
8075
8076 return !!oo_objects(s->oo);
8077}
8078
8079static void list_slab_objects(struct kmem_cache *s, struct slab *slab)
8080{
8081#ifdef CONFIG_SLUB_DEBUG
8082 void *addr = slab_address(slab);
8083 void *p;
8084
8085 if (!slab_add_kunit_errors())
8086 slab_bug(s, "Objects remaining on __kmem_cache_shutdown()");
8087
8088 spin_lock(&object_map_lock);
8089 __fill_map(object_map, s, slab);
8090
8091 for_each_object(p, s, addr, slab->objects) {
8092
8093 if (!test_bit(__obj_to_index(s, addr, p), object_map)) {
8094 if (slab_add_kunit_errors())
8095 continue;
8096 pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
8097 print_tracking(s, p);
8098 }
8099 }
8100 spin_unlock(&object_map_lock);
8101
8102 __slab_err(slab);
8103#endif
8104}
8105
8106/*
8107 * Attempt to free all partial slabs on a node.
8108 * This is called from __kmem_cache_shutdown(). We must take list_lock
8109 * because sysfs file might still access partial list after the shutdowning.
8110 */
8111static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
8112{
8113 LIST_HEAD(discard);
8114 struct slab *slab, *h;
8115
8116 BUG_ON(irqs_disabled());
8117 spin_lock_irq(&n->list_lock);
8118 list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
8119 if (!slab->inuse) {
8120 remove_partial(n, slab);
8121 list_add(&slab->slab_list, &discard);
8122 } else {
8123 list_slab_objects(s, slab);
8124 }
8125 }
8126 spin_unlock_irq(&n->list_lock);
8127
8128 list_for_each_entry_safe(slab, h, &discard, slab_list)
8129 discard_slab(s, slab);
8130}
8131
8132bool __kmem_cache_empty(struct kmem_cache *s)
8133{
8134 int node;
8135 struct kmem_cache_node *n;
8136
8137 for_each_kmem_cache_node(s, node, n)
8138 if (n->nr_partial || node_nr_slabs(n))
8139 return false;
8140 return true;
8141}
8142
8143/*
8144 * Release all resources used by a slab cache.
8145 */
8146int __kmem_cache_shutdown(struct kmem_cache *s)
8147{
8148 int node;
8149 struct kmem_cache_node *n;
8150
8151 flush_all_cpus_locked(s);
8152
8153 /* we might have rcu sheaves in flight */
8154 if (cache_has_sheaves(s))
8155 rcu_barrier();
8156
8157 for_each_node(node) {
8158 struct node_barn *barn = get_barn_node(s, node);
8159
8160 if (barn)
8161 barn_shrink(s, barn);
8162 }
8163
8164 /* Attempt to free all objects */
8165 for_each_kmem_cache_node(s, node, n) {
8166 free_partial(s, n);
8167 if (n->nr_partial || node_nr_slabs(n))
8168 return 1;
8169 }
8170 return 0;
8171}
8172
8173#ifdef CONFIG_PRINTK
8174void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
8175{
8176 void *base;
8177 int __maybe_unused i;
8178 unsigned int objnr;
8179 void *objp;
8180 void *objp0;
8181 struct kmem_cache *s = slab->slab_cache;
8182 struct track __maybe_unused *trackp;
8183
8184 kpp->kp_ptr = object;
8185 kpp->kp_slab = slab;
8186 kpp->kp_slab_cache = s;
8187 base = slab_address(slab);
8188 objp0 = kasan_reset_tag(object);
8189#ifdef CONFIG_SLUB_DEBUG
8190 objp = restore_red_left(s, objp0);
8191#else
8192 objp = objp0;
8193#endif
8194 objnr = obj_to_index(s, slab, objp);
8195 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
8196 objp = base + s->size * objnr;
8197 kpp->kp_objp = objp;
8198 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
8199 || (objp - base) % s->size) ||
8200 !(s->flags & SLAB_STORE_USER))
8201 return;
8202#ifdef CONFIG_SLUB_DEBUG
8203 objp = fixup_red_left(s, objp);
8204 trackp = get_track(s, objp, TRACK_ALLOC);
8205 kpp->kp_ret = (void *)trackp->addr;
8206#ifdef CONFIG_STACKDEPOT
8207 {
8208 depot_stack_handle_t handle;
8209 unsigned long *entries;
8210 unsigned int nr_entries;
8211
8212 handle = READ_ONCE(trackp->handle);
8213 if (handle) {
8214 nr_entries = stack_depot_fetch(handle, &entries);
8215 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
8216 kpp->kp_stack[i] = (void *)entries[i];
8217 }
8218
8219 trackp = get_track(s, objp, TRACK_FREE);
8220 handle = READ_ONCE(trackp->handle);
8221 if (handle) {
8222 nr_entries = stack_depot_fetch(handle, &entries);
8223 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
8224 kpp->kp_free_stack[i] = (void *)entries[i];
8225 }
8226 }
8227#endif
8228#endif
8229}
8230#endif
8231
8232/********************************************************************
8233 * Kmalloc subsystem
8234 *******************************************************************/
8235
8236static int __init setup_slub_min_order(const char *str, const struct kernel_param *kp)
8237{
8238 int ret;
8239
8240 ret = kstrtouint(str, 0, &slub_min_order);
8241 if (ret)
8242 return ret;
8243
8244 if (slub_min_order > slub_max_order)
8245 slub_max_order = slub_min_order;
8246
8247 return 0;
8248}
8249
8250static const struct kernel_param_ops param_ops_slab_min_order __initconst = {
8251 .set = setup_slub_min_order,
8252};
8253__core_param_cb(slab_min_order, &param_ops_slab_min_order, &slub_min_order, 0);
8254__core_param_cb(slub_min_order, &param_ops_slab_min_order, &slub_min_order, 0);
8255
8256static int __init setup_slub_max_order(const char *str, const struct kernel_param *kp)
8257{
8258 int ret;
8259
8260 ret = kstrtouint(str, 0, &slub_max_order);
8261 if (ret)
8262 return ret;
8263
8264 slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER);
8265
8266 if (slub_min_order > slub_max_order)
8267 slub_min_order = slub_max_order;
8268
8269 return 0;
8270}
8271
8272static const struct kernel_param_ops param_ops_slab_max_order __initconst = {
8273 .set = setup_slub_max_order,
8274};
8275__core_param_cb(slab_max_order, &param_ops_slab_max_order, &slub_max_order, 0);
8276__core_param_cb(slub_max_order, &param_ops_slab_max_order, &slub_max_order, 0);
8277
8278core_param(slab_min_objects, slub_min_objects, uint, 0);
8279core_param(slub_min_objects, slub_min_objects, uint, 0);
8280
8281#ifdef CONFIG_NUMA
8282static int __init setup_slab_strict_numa(const char *str, const struct kernel_param *kp)
8283{
8284 if (nr_node_ids > 1) {
8285 static_branch_enable(&strict_numa);
8286 pr_info("SLUB: Strict NUMA enabled.\n");
8287 } else {
8288 pr_warn("slab_strict_numa parameter set on non NUMA system.\n");
8289 }
8290
8291 return 0;
8292}
8293
8294static const struct kernel_param_ops param_ops_slab_strict_numa __initconst = {
8295 .flags = KERNEL_PARAM_OPS_FL_NOARG,
8296 .set = setup_slab_strict_numa,
8297};
8298__core_param_cb(slab_strict_numa, &param_ops_slab_strict_numa, NULL, 0);
8299#endif
8300
8301
8302#ifdef CONFIG_HARDENED_USERCOPY
8303/*
8304 * Rejects incorrectly sized objects and objects that are to be copied
8305 * to/from userspace but do not fall entirely within the containing slab
8306 * cache's usercopy region.
8307 *
8308 * Returns NULL if check passes, otherwise const char * to name of cache
8309 * to indicate an error.
8310 */
8311void __check_heap_object(const void *ptr, unsigned long n,
8312 const struct slab *slab, bool to_user)
8313{
8314 struct kmem_cache *s;
8315 unsigned int offset;
8316 bool is_kfence = is_kfence_address(ptr);
8317
8318 ptr = kasan_reset_tag(ptr);
8319
8320 /* Find object and usable object size. */
8321 s = slab->slab_cache;
8322
8323 /* Reject impossible pointers. */
8324 if (ptr < slab_address(slab))
8325 usercopy_abort("SLUB object not in SLUB page?!", NULL,
8326 to_user, 0, n);
8327
8328 /* Find offset within object. */
8329 if (is_kfence)
8330 offset = ptr - kfence_object_start(ptr);
8331 else
8332 offset = (ptr - slab_address(slab)) % s->size;
8333
8334 /* Adjust for redzone and reject if within the redzone. */
8335 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
8336 if (offset < s->red_left_pad)
8337 usercopy_abort("SLUB object in left red zone",
8338 s->name, to_user, offset, n);
8339 offset -= s->red_left_pad;
8340 }
8341
8342 /* Allow address range falling entirely within usercopy region. */
8343 if (offset >= s->useroffset &&
8344 offset - s->useroffset <= s->usersize &&
8345 n <= s->useroffset - offset + s->usersize)
8346 return;
8347
8348 usercopy_abort("SLUB object", s->name, to_user, offset, n);
8349}
8350#endif /* CONFIG_HARDENED_USERCOPY */
8351
8352#define SHRINK_PROMOTE_MAX 32
8353
8354/*
8355 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
8356 * up most to the head of the partial lists. New allocations will then
8357 * fill those up and thus they can be removed from the partial lists.
8358 *
8359 * The slabs with the least items are placed last. This results in them
8360 * being allocated from last increasing the chance that the last objects
8361 * are freed in them.
8362 */
8363static int __kmem_cache_do_shrink(struct kmem_cache *s)
8364{
8365 int node;
8366 int i;
8367 struct kmem_cache_node *n;
8368 struct slab *slab;
8369 struct slab *t;
8370 struct list_head discard;
8371 struct list_head promote[SHRINK_PROMOTE_MAX];
8372 unsigned long flags;
8373 int ret = 0;
8374
8375 for_each_node(node) {
8376 struct node_barn *barn = get_barn_node(s, node);
8377
8378 if (barn)
8379 barn_shrink(s, barn);
8380 }
8381
8382 for_each_kmem_cache_node(s, node, n) {
8383 INIT_LIST_HEAD(&discard);
8384 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
8385 INIT_LIST_HEAD(promote + i);
8386
8387 spin_lock_irqsave(&n->list_lock, flags);
8388
8389 /*
8390 * Build lists of slabs to discard or promote.
8391 *
8392 * Note that concurrent frees may occur while we hold the
8393 * list_lock. slab->inuse here is the upper limit.
8394 */
8395 list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
8396 int free = slab->objects - slab->inuse;
8397
8398 /* Do not reread slab->inuse */
8399 barrier();
8400
8401 /* We do not keep full slabs on the list */
8402 BUG_ON(free <= 0);
8403
8404 if (free == slab->objects) {
8405 list_move(&slab->slab_list, &discard);
8406 clear_node_partial_state(n, slab);
8407 dec_slabs_node(s, node, slab->objects);
8408 } else if (free <= SHRINK_PROMOTE_MAX)
8409 list_move(&slab->slab_list, promote + free - 1);
8410 }
8411
8412 /*
8413 * Promote the slabs filled up most to the head of the
8414 * partial list.
8415 */
8416 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
8417 list_splice(promote + i, &n->partial);
8418
8419 spin_unlock_irqrestore(&n->list_lock, flags);
8420
8421 /* Release empty slabs */
8422 list_for_each_entry_safe(slab, t, &discard, slab_list)
8423 free_slab(s, slab);
8424
8425 if (node_nr_slabs(n))
8426 ret = 1;
8427 }
8428
8429 return ret;
8430}
8431
8432int __kmem_cache_shrink(struct kmem_cache *s)
8433{
8434 flush_all(s);
8435 return __kmem_cache_do_shrink(s);
8436}
8437
8438static int slab_mem_going_offline_callback(void)
8439{
8440 struct kmem_cache *s;
8441
8442 mutex_lock(&slab_mutex);
8443 list_for_each_entry(s, &slab_caches, list) {
8444 flush_all_cpus_locked(s);
8445 __kmem_cache_do_shrink(s);
8446 }
8447 mutex_unlock(&slab_mutex);
8448
8449 return 0;
8450}
8451
8452static int slab_mem_going_online_callback(int nid)
8453{
8454 struct kmem_cache_node *n;
8455 struct kmem_cache *s;
8456 int ret = 0;
8457
8458 /*
8459 * We are bringing a node online. No memory is available yet. We must
8460 * allocate a kmem_cache_node structure in order to bring the node
8461 * online.
8462 */
8463 mutex_lock(&slab_mutex);
8464 list_for_each_entry(s, &slab_caches, list) {
8465 struct node_barn *barn = NULL;
8466
8467 /*
8468 * The structure may already exist if the node was previously
8469 * onlined and offlined.
8470 */
8471 if (get_node(s, nid))
8472 continue;
8473
8474 if (cache_has_sheaves(s) && !get_barn_node(s, nid)) {
8475
8476 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, nid);
8477
8478 if (!barn) {
8479 ret = -ENOMEM;
8480 goto out;
8481 }
8482 }
8483
8484 /*
8485 * XXX: kmem_cache_alloc_node will fallback to other nodes
8486 * since memory is not yet available from the node that
8487 * is brought up.
8488 */
8489 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
8490 if (!n) {
8491 kfree(barn);
8492 ret = -ENOMEM;
8493 goto out;
8494 }
8495
8496 init_kmem_cache_node(n);
8497 s->per_node[nid].node = n;
8498
8499 if (barn) {
8500 barn_init(barn);
8501 s->per_node[nid].barn = barn;
8502 }
8503 }
8504 /*
8505 * Any cache created after this point will also have kmem_cache_node
8506 * and barn initialized for the new node.
8507 */
8508 node_set(nid, slab_nodes);
8509 node_set(nid, slab_barn_nodes);
8510out:
8511 mutex_unlock(&slab_mutex);
8512 return ret;
8513}
8514
8515static int slab_memory_callback(struct notifier_block *self,
8516 unsigned long action, void *arg)
8517{
8518 struct node_notify *nn = arg;
8519 int nid = nn->nid;
8520 int ret = 0;
8521
8522 switch (action) {
8523 case NODE_ADDING_FIRST_MEMORY:
8524 ret = slab_mem_going_online_callback(nid);
8525 break;
8526 case NODE_REMOVING_LAST_MEMORY:
8527 ret = slab_mem_going_offline_callback();
8528 break;
8529 }
8530 if (ret)
8531 ret = notifier_from_errno(ret);
8532 else
8533 ret = NOTIFY_OK;
8534 return ret;
8535}
8536
8537/********************************************************************
8538 * Basic setup of slabs
8539 *******************************************************************/
8540
8541/*
8542 * Used for early kmem_cache structures that were allocated using
8543 * the page allocator. Allocate them properly then fix up the pointers
8544 * that may be pointing to the wrong kmem_cache structure.
8545 */
8546
8547static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
8548{
8549 int node;
8550 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
8551 struct kmem_cache_node *n;
8552
8553 memcpy(s, static_cache, kmem_cache->object_size);
8554
8555 for_each_kmem_cache_node(s, node, n) {
8556 struct slab *p;
8557
8558 list_for_each_entry(p, &n->partial, slab_list)
8559 p->slab_cache = s;
8560
8561#ifdef CONFIG_SLUB_DEBUG
8562 list_for_each_entry(p, &n->full, slab_list)
8563 p->slab_cache = s;
8564#endif
8565 }
8566 list_add(&s->list, &slab_caches);
8567 return s;
8568}
8569
8570/*
8571 * Finish the sheaves initialization done normally by init_percpu_sheaves() and
8572 * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
8573 * since sheaves and barns are allocated by kmalloc.
8574 */
8575static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
8576{
8577 struct kmem_cache_args empty_args = {};
8578 unsigned int capacity;
8579 bool failed = false;
8580 int node, cpu;
8581
8582 VM_WARN_ON_ONCE(cache_has_sheaves(s));
8583
8584 capacity = calculate_sheaf_capacity(s, &empty_args);
8585
8586 /* capacity can be 0 due to debugging or SLUB_TINY */
8587 if (!capacity)
8588 return;
8589
8590 for_each_node_mask(node, slab_barn_nodes) {
8591 struct node_barn *barn;
8592
8593 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
8594
8595 if (!barn) {
8596 failed = true;
8597 goto out;
8598 }
8599
8600 barn_init(barn);
8601 s->per_node[node].barn = barn;
8602 }
8603
8604 for_each_possible_cpu(cpu) {
8605 struct slub_percpu_sheaves *pcs;
8606
8607 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
8608
8609 pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL,
8610 SLAB_ALLOC_DEFAULT, capacity);
8611
8612 if (!pcs->main) {
8613 failed = true;
8614 break;
8615 }
8616 }
8617
8618out:
8619 /*
8620 * It's still early in boot so treat this like same as a failure to
8621 * create the kmalloc cache in the first place
8622 */
8623 if (failed)
8624 panic("Out of memory when creating kmem_cache %s\n", s->name);
8625
8626 s->sheaf_capacity = capacity;
8627}
8628
8629static void __init bootstrap_kmalloc_sheaves(void)
8630{
8631 enum kmalloc_cache_type type;
8632
8633 for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
8634 for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
8635 struct kmem_cache *s = kmalloc_caches[type][idx];
8636
8637 /* Do not bootstrap twice when caches are aliased */
8638 if (s && !cache_has_sheaves(s))
8639 bootstrap_cache_sheaves(s);
8640 }
8641 }
8642}
8643
8644void __init kmem_cache_init(void)
8645{
8646 static __initdata struct kmem_cache boot_kmem_cache,
8647 boot_kmem_cache_node;
8648 int node;
8649
8650 slab_obj_ext_has_codetag_init();
8651
8652 if (debug_guardpage_minorder())
8653 slub_max_order = 0;
8654
8655 /* Inform pointer hashing choice about slub debugging state. */
8656 hash_pointers_finalize(__slub_debug_enabled());
8657
8658 kmem_cache_node = &boot_kmem_cache_node;
8659 kmem_cache = &boot_kmem_cache;
8660
8661 /*
8662 * Initialize the nodemask for which we will allocate per node
8663 * structures. Here we don't need taking slab_mutex yet.
8664 */
8665 for_each_node_state(node, N_MEMORY)
8666 node_set(node, slab_nodes);
8667
8668 for_each_online_node(node)
8669 node_set(node, slab_barn_nodes);
8670
8671 create_boot_cache(kmem_cache_node, "kmem_cache_node",
8672 sizeof(struct kmem_cache_node),
8673 SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT,
8674 0, 0);
8675
8676 hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
8677
8678 /* Able to allocate the per node structures */
8679 slab_state = PARTIAL;
8680
8681 create_boot_cache(kmem_cache, "kmem_cache",
8682 offsetof(struct kmem_cache, per_node) +
8683 nr_node_ids * sizeof(struct kmem_cache_per_node_ptrs),
8684 SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT,
8685 0, 0);
8686
8687 kmem_cache = bootstrap(&boot_kmem_cache);
8688 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
8689
8690 /* Now we can use the kmem_cache to allocate kmalloc slabs */
8691 setup_kmalloc_cache_index_table();
8692 create_kmalloc_caches();
8693
8694 bootstrap_kmalloc_sheaves();
8695
8696 /* Setup random freelists for each cache */
8697 init_freelist_randomization();
8698
8699 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", slub_cpu_setup,
8700 slub_cpu_dead);
8701
8702 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
8703 cache_line_size(),
8704 slub_min_order, slub_max_order, slub_min_objects,
8705 nr_cpu_ids, nr_node_ids);
8706}
8707
8708void __init kmem_cache_init_late(void)
8709{
8710 flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM | WQ_PERCPU,
8711 0);
8712 WARN_ON(!flushwq);
8713#ifdef CONFIG_SLAB_FREELIST_RANDOM
8714 prandom_init_once(&slab_rnd_state);
8715#endif
8716}
8717
8718int do_kmem_cache_create(struct kmem_cache *s, const char *name,
8719 unsigned int size, struct kmem_cache_args *args,
8720 slab_flags_t flags)
8721{
8722 int err = -EINVAL;
8723
8724 s->name = name;
8725 s->size = s->object_size = size;
8726
8727 s->flags = kmem_cache_flags(flags, s->name);
8728#ifdef CONFIG_SLAB_FREELIST_HARDENED
8729 s->random = get_random_long();
8730#endif
8731 s->align = args->align;
8732 s->ctor = args->ctor;
8733#ifdef CONFIG_HARDENED_USERCOPY
8734 s->useroffset = args->useroffset;
8735 s->usersize = args->usersize;
8736#endif
8737
8738 if (!calculate_sizes(args, s))
8739 goto out;
8740 if (disable_higher_order_debug) {
8741 /*
8742 * Disable debugging flags that store metadata if the min slab
8743 * order increased.
8744 */
8745 if (get_order(s->size) > get_order(s->object_size)) {
8746 s->flags &= ~DEBUG_METADATA_FLAGS;
8747 s->offset = 0;
8748 if (!calculate_sizes(args, s))
8749 goto out;
8750 }
8751 }
8752
8753#ifdef system_has_freelist_aba
8754 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
8755 /* Enable fast mode */
8756 s->flags |= __CMPXCHG_DOUBLE;
8757 }
8758#endif
8759
8760 /*
8761 * The larger the object size is, the more slabs we want on the partial
8762 * list to avoid pounding the page allocator excessively.
8763 */
8764 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
8765 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
8766
8767 s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves);
8768 if (!s->cpu_sheaves) {
8769 err = -ENOMEM;
8770 goto out;
8771 }
8772
8773#ifdef CONFIG_NUMA
8774 s->remote_node_defrag_ratio = 1000;
8775#endif
8776
8777 /* Initialize the pre-computed randomized freelist if slab is up */
8778 if (slab_state >= UP) {
8779 if (init_cache_random_seq(s))
8780 goto out;
8781 }
8782
8783 if (!init_kmem_cache_nodes(s))
8784 goto out;
8785
8786#ifdef CONFIG_SLUB_STATS
8787 if (!alloc_kmem_cache_stats(s))
8788 goto out;
8789#endif
8790
8791 err = init_percpu_sheaves(s);
8792 if (err)
8793 goto out;
8794
8795 err = 0;
8796
8797 /* Mutex is not taken during early boot */
8798 if (slab_state <= UP)
8799 goto out;
8800
8801 /*
8802 * Failing to create sysfs files is not critical to SLUB functionality.
8803 * If it fails, proceed with cache creation without these files.
8804 */
8805 if (sysfs_slab_add(s))
8806 pr_err("SLUB: Unable to add cache %s to sysfs\n", s->name);
8807
8808 if (s->flags & SLAB_STORE_USER)
8809 debugfs_slab_add(s);
8810
8811out:
8812 if (err)
8813 __kmem_cache_release(s);
8814 return err;
8815}
8816
8817#ifdef SLAB_SUPPORTS_SYSFS
8818static int count_inuse(struct slab *slab)
8819{
8820 return slab->inuse;
8821}
8822
8823static int count_total(struct slab *slab)
8824{
8825 return slab->objects;
8826}
8827#endif
8828
8829#ifdef CONFIG_SLUB_DEBUG
8830static void validate_slab(struct kmem_cache *s, struct slab *slab,
8831 unsigned long *obj_map)
8832{
8833 void *p;
8834 void *addr = slab_address(slab);
8835
8836 if (!validate_slab_ptr(slab)) {
8837 slab_err(s, slab, "Not a valid slab page");
8838 return;
8839 }
8840
8841 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
8842 return;
8843
8844 /* Now we know that a valid freelist exists */
8845 __fill_map(obj_map, s, slab);
8846 for_each_object(p, s, addr, slab->objects) {
8847 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
8848 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
8849
8850 if (!check_object(s, slab, p, val))
8851 break;
8852 }
8853}
8854
8855static int validate_slab_node(struct kmem_cache *s,
8856 struct kmem_cache_node *n, unsigned long *obj_map)
8857{
8858 unsigned long count = 0;
8859 struct slab *slab;
8860 unsigned long flags;
8861
8862 spin_lock_irqsave(&n->list_lock, flags);
8863
8864 list_for_each_entry(slab, &n->partial, slab_list) {
8865 validate_slab(s, slab, obj_map);
8866 count++;
8867 }
8868 if (count != n->nr_partial) {
8869 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
8870 s->name, count, n->nr_partial);
8871 slab_add_kunit_errors();
8872 }
8873
8874 if (!(s->flags & SLAB_STORE_USER))
8875 goto out;
8876
8877 list_for_each_entry(slab, &n->full, slab_list) {
8878 validate_slab(s, slab, obj_map);
8879 count++;
8880 }
8881 if (count != node_nr_slabs(n)) {
8882 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
8883 s->name, count, node_nr_slabs(n));
8884 slab_add_kunit_errors();
8885 }
8886
8887out:
8888 spin_unlock_irqrestore(&n->list_lock, flags);
8889 return count;
8890}
8891
8892long validate_slab_cache(struct kmem_cache *s)
8893{
8894 int node;
8895 unsigned long count = 0;
8896 struct kmem_cache_node *n;
8897 unsigned long *obj_map;
8898
8899 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
8900 if (!obj_map)
8901 return -ENOMEM;
8902
8903 flush_all(s);
8904 for_each_kmem_cache_node(s, node, n)
8905 count += validate_slab_node(s, n, obj_map);
8906
8907 bitmap_free(obj_map);
8908
8909 return count;
8910}
8911EXPORT_SYMBOL(validate_slab_cache);
8912
8913#ifdef CONFIG_DEBUG_FS
8914/*
8915 * Generate lists of code addresses where slabcache objects are allocated
8916 * and freed.
8917 */
8918
8919struct location {
8920 depot_stack_handle_t handle;
8921 unsigned long count;
8922 unsigned long addr;
8923 unsigned long waste;
8924 long long sum_time;
8925 long min_time;
8926 long max_time;
8927 long min_pid;
8928 long max_pid;
8929 DECLARE_BITMAP(cpus, NR_CPUS);
8930 nodemask_t nodes;
8931};
8932
8933struct loc_track {
8934 unsigned long max;
8935 unsigned long count;
8936 struct location *loc;
8937 loff_t idx;
8938};
8939
8940static struct dentry *slab_debugfs_root;
8941
8942static void free_loc_track(struct loc_track *t)
8943{
8944 if (t->max)
8945 free_pages((unsigned long)t->loc,
8946 get_order(sizeof(struct location) * t->max));
8947}
8948
8949static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
8950{
8951 struct location *l;
8952 int order;
8953
8954 order = get_order(sizeof(struct location) * max);
8955
8956 l = (void *)__get_free_pages(flags, order);
8957 if (!l)
8958 return 0;
8959
8960 if (t->count) {
8961 memcpy(l, t->loc, sizeof(struct location) * t->count);
8962 free_loc_track(t);
8963 }
8964 t->max = max;
8965 t->loc = l;
8966 return 1;
8967}
8968
8969static int add_location(struct loc_track *t, struct kmem_cache *s,
8970 const struct track *track,
8971 unsigned int orig_size)
8972{
8973 long start, end, pos;
8974 struct location *l;
8975 unsigned long caddr, chandle, cwaste;
8976 unsigned long age = jiffies - track->when;
8977 depot_stack_handle_t handle = 0;
8978 unsigned int waste = s->object_size - orig_size;
8979
8980#ifdef CONFIG_STACKDEPOT
8981 handle = READ_ONCE(track->handle);
8982#endif
8983 start = -1;
8984 end = t->count;
8985
8986 for ( ; ; ) {
8987 pos = start + (end - start + 1) / 2;
8988
8989 /*
8990 * There is nothing at "end". If we end up there
8991 * we need to add something to before end.
8992 */
8993 if (pos == end)
8994 break;
8995
8996 l = &t->loc[pos];
8997 caddr = l->addr;
8998 chandle = l->handle;
8999 cwaste = l->waste;
9000 if ((track->addr == caddr) && (handle == chandle) &&
9001 (waste == cwaste)) {
9002
9003 l->count++;
9004 if (track->when) {
9005 l->sum_time += age;
9006 if (age < l->min_time)
9007 l->min_time = age;
9008 if (age > l->max_time)
9009 l->max_time = age;
9010
9011 if (track->pid < l->min_pid)
9012 l->min_pid = track->pid;
9013 if (track->pid > l->max_pid)
9014 l->max_pid = track->pid;
9015
9016 cpumask_set_cpu(track->cpu,
9017 to_cpumask(l->cpus));
9018 }
9019 node_set(page_to_nid(virt_to_page(track)), l->nodes);
9020 return 1;
9021 }
9022
9023 if (track->addr < caddr)
9024 end = pos;
9025 else if (track->addr == caddr && handle < chandle)
9026 end = pos;
9027 else if (track->addr == caddr && handle == chandle &&
9028 waste < cwaste)
9029 end = pos;
9030 else
9031 start = pos;
9032 }
9033
9034 /*
9035 * Not found. Insert new tracking element.
9036 */
9037 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
9038 return 0;
9039
9040 l = t->loc + pos;
9041 if (pos < t->count)
9042 memmove(l + 1, l,
9043 (t->count - pos) * sizeof(struct location));
9044 t->count++;
9045 l->count = 1;
9046 l->addr = track->addr;
9047 l->sum_time = age;
9048 l->min_time = age;
9049 l->max_time = age;
9050 l->min_pid = track->pid;
9051 l->max_pid = track->pid;
9052 l->handle = handle;
9053 l->waste = waste;
9054 cpumask_clear(to_cpumask(l->cpus));
9055 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
9056 nodes_clear(l->nodes);
9057 node_set(page_to_nid(virt_to_page(track)), l->nodes);
9058 return 1;
9059}
9060
9061static void process_slab(struct loc_track *t, struct kmem_cache *s,
9062 struct slab *slab, enum track_item alloc,
9063 unsigned long *obj_map)
9064{
9065 void *addr = slab_address(slab);
9066 bool is_alloc = (alloc == TRACK_ALLOC);
9067 void *p;
9068
9069 __fill_map(obj_map, s, slab);
9070
9071 for_each_object(p, s, addr, slab->objects)
9072 if (!test_bit(__obj_to_index(s, addr, p), obj_map))
9073 add_location(t, s, get_track(s, p, alloc),
9074 is_alloc ? get_orig_size(s, p) :
9075 s->object_size);
9076}
9077#endif /* CONFIG_DEBUG_FS */
9078#endif /* CONFIG_SLUB_DEBUG */
9079
9080#ifdef SLAB_SUPPORTS_SYSFS
9081enum slab_stat_type {
9082 SL_ALL, /* All slabs */
9083 SL_PARTIAL, /* Only partially allocated slabs */
9084 SL_OBJECTS, /* Determine allocated objects not slabs */
9085 SL_TOTAL /* Determine object capacity not slabs */
9086};
9087
9088#define SO_ALL (1 << SL_ALL)
9089#define SO_PARTIAL (1 << SL_PARTIAL)
9090#define SO_OBJECTS (1 << SL_OBJECTS)
9091#define SO_TOTAL (1 << SL_TOTAL)
9092
9093static ssize_t show_slab_objects(struct kmem_cache *s,
9094 char *buf, unsigned long flags)
9095{
9096 unsigned long total = 0;
9097 int node;
9098 int x;
9099 unsigned long *nodes;
9100 int len = 0;
9101
9102 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
9103 if (!nodes)
9104 return -ENOMEM;
9105
9106 /*
9107 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
9108 * already held which will conflict with an existing lock order:
9109 *
9110 * mem_hotplug_lock->slab_mutex->kernfs_mutex
9111 *
9112 * We don't really need mem_hotplug_lock (to hold off
9113 * slab_mem_going_offline_callback) here because slab's memory hot
9114 * unplug code doesn't destroy the kmem_cache->node[] data.
9115 */
9116
9117#ifdef CONFIG_SLUB_DEBUG
9118 if (flags & SO_ALL) {
9119 struct kmem_cache_node *n;
9120
9121 for_each_kmem_cache_node(s, node, n) {
9122
9123 if (flags & SO_TOTAL)
9124 x = node_nr_objs(n);
9125 else if (flags & SO_OBJECTS)
9126 x = node_nr_objs(n) - count_partial(n, count_free);
9127 else
9128 x = node_nr_slabs(n);
9129 total += x;
9130 nodes[node] += x;
9131 }
9132
9133 } else
9134#endif
9135 if (flags & SO_PARTIAL) {
9136 struct kmem_cache_node *n;
9137
9138 for_each_kmem_cache_node(s, node, n) {
9139 if (flags & SO_TOTAL)
9140 x = count_partial(n, count_total);
9141 else if (flags & SO_OBJECTS)
9142 x = count_partial(n, count_inuse);
9143 else
9144 x = n->nr_partial;
9145 total += x;
9146 nodes[node] += x;
9147 }
9148 }
9149
9150 len += sysfs_emit_at(buf, len, "%lu", total);
9151#ifdef CONFIG_NUMA
9152 for (node = 0; node < nr_node_ids; node++) {
9153 if (nodes[node])
9154 len += sysfs_emit_at(buf, len, " N%d=%lu",
9155 node, nodes[node]);
9156 }
9157#endif
9158 len += sysfs_emit_at(buf, len, "\n");
9159 kfree(nodes);
9160
9161 return len;
9162}
9163
9164#define to_slab_attr(n) container_of_const(n, struct slab_attribute, attr)
9165#define to_slab(n) container_of(n, struct kmem_cache, kobj)
9166
9167struct slab_attribute {
9168 struct attribute attr;
9169 ssize_t (*show)(struct kmem_cache *s, char *buf);
9170 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
9171};
9172
9173#define SLAB_ATTR_RO(_name) \
9174 static const struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400)
9175
9176#define SLAB_ATTR(_name) \
9177 static const struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600)
9178
9179static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
9180{
9181 return sysfs_emit(buf, "%u\n", s->size);
9182}
9183SLAB_ATTR_RO(slab_size);
9184
9185static ssize_t align_show(struct kmem_cache *s, char *buf)
9186{
9187 return sysfs_emit(buf, "%u\n", s->align);
9188}
9189SLAB_ATTR_RO(align);
9190
9191static ssize_t object_size_show(struct kmem_cache *s, char *buf)
9192{
9193 return sysfs_emit(buf, "%u\n", s->object_size);
9194}
9195SLAB_ATTR_RO(object_size);
9196
9197static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
9198{
9199 return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
9200}
9201SLAB_ATTR_RO(objs_per_slab);
9202
9203static ssize_t order_show(struct kmem_cache *s, char *buf)
9204{
9205 return sysfs_emit(buf, "%u\n", oo_order(s->oo));
9206}
9207SLAB_ATTR_RO(order);
9208
9209static ssize_t sheaf_capacity_show(struct kmem_cache *s, char *buf)
9210{
9211 return sysfs_emit(buf, "%u\n", s->sheaf_capacity);
9212}
9213SLAB_ATTR_RO(sheaf_capacity);
9214
9215static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
9216{
9217 return sysfs_emit(buf, "%lu\n", s->min_partial);
9218}
9219
9220static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
9221 size_t length)
9222{
9223 unsigned long min;
9224 int err;
9225
9226 err = kstrtoul(buf, 10, &min);
9227 if (err)
9228 return err;
9229
9230 s->min_partial = min;
9231 return length;
9232}
9233SLAB_ATTR(min_partial);
9234
9235static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
9236{
9237 return sysfs_emit(buf, "0\n");
9238}
9239
9240static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
9241 size_t length)
9242{
9243 unsigned int objects;
9244 int err;
9245
9246 err = kstrtouint(buf, 10, &objects);
9247 if (err)
9248 return err;
9249 if (objects)
9250 return -EINVAL;
9251
9252 return length;
9253}
9254SLAB_ATTR(cpu_partial);
9255
9256static ssize_t ctor_show(struct kmem_cache *s, char *buf)
9257{
9258 if (!s->ctor)
9259 return 0;
9260 return sysfs_emit(buf, "%pS\n", s->ctor);
9261}
9262SLAB_ATTR_RO(ctor);
9263
9264static ssize_t aliases_show(struct kmem_cache *s, char *buf)
9265{
9266 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
9267}
9268SLAB_ATTR_RO(aliases);
9269
9270static ssize_t partial_show(struct kmem_cache *s, char *buf)
9271{
9272 return show_slab_objects(s, buf, SO_PARTIAL);
9273}
9274SLAB_ATTR_RO(partial);
9275
9276static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
9277{
9278 return sysfs_emit(buf, "0\n");
9279}
9280SLAB_ATTR_RO(cpu_slabs);
9281
9282static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
9283{
9284 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
9285}
9286SLAB_ATTR_RO(objects_partial);
9287
9288static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
9289{
9290 return sysfs_emit(buf, "0(0)\n");
9291}
9292SLAB_ATTR_RO(slabs_cpu_partial);
9293
9294static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
9295{
9296 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
9297}
9298SLAB_ATTR_RO(reclaim_account);
9299
9300static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
9301{
9302 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
9303}
9304SLAB_ATTR_RO(hwcache_align);
9305
9306#ifdef CONFIG_ZONE_DMA
9307static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
9308{
9309 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
9310}
9311SLAB_ATTR_RO(cache_dma);
9312#endif
9313
9314#ifdef CONFIG_HARDENED_USERCOPY
9315static ssize_t usersize_show(struct kmem_cache *s, char *buf)
9316{
9317 return sysfs_emit(buf, "%u\n", s->usersize);
9318}
9319SLAB_ATTR_RO(usersize);
9320#endif
9321
9322static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
9323{
9324 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
9325}
9326SLAB_ATTR_RO(destroy_by_rcu);
9327
9328#ifdef CONFIG_SLUB_DEBUG
9329static ssize_t slabs_show(struct kmem_cache *s, char *buf)
9330{
9331 return show_slab_objects(s, buf, SO_ALL);
9332}
9333SLAB_ATTR_RO(slabs);
9334
9335static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
9336{
9337 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
9338}
9339SLAB_ATTR_RO(total_objects);
9340
9341static ssize_t objects_show(struct kmem_cache *s, char *buf)
9342{
9343 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
9344}
9345SLAB_ATTR_RO(objects);
9346
9347static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
9348{
9349 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
9350}
9351SLAB_ATTR_RO(sanity_checks);
9352
9353static ssize_t trace_show(struct kmem_cache *s, char *buf)
9354{
9355 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
9356}
9357SLAB_ATTR_RO(trace);
9358
9359static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
9360{
9361 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
9362}
9363
9364SLAB_ATTR_RO(red_zone);
9365
9366static ssize_t poison_show(struct kmem_cache *s, char *buf)
9367{
9368 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
9369}
9370
9371SLAB_ATTR_RO(poison);
9372
9373static ssize_t store_user_show(struct kmem_cache *s, char *buf)
9374{
9375 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
9376}
9377
9378SLAB_ATTR_RO(store_user);
9379
9380static ssize_t validate_show(struct kmem_cache *s, char *buf)
9381{
9382 return 0;
9383}
9384
9385static ssize_t validate_store(struct kmem_cache *s,
9386 const char *buf, size_t length)
9387{
9388 int ret = -EINVAL;
9389
9390 if (buf[0] == '1' && kmem_cache_debug(s)) {
9391 ret = validate_slab_cache(s);
9392 if (ret >= 0)
9393 ret = length;
9394 }
9395 return ret;
9396}
9397SLAB_ATTR(validate);
9398
9399#endif /* CONFIG_SLUB_DEBUG */
9400
9401#ifdef CONFIG_FAILSLAB
9402static ssize_t failslab_show(struct kmem_cache *s, char *buf)
9403{
9404 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
9405}
9406
9407static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
9408 size_t length)
9409{
9410 if (s->refcount > 1)
9411 return -EINVAL;
9412
9413 if (buf[0] == '1')
9414 WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB);
9415 else
9416 WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB);
9417
9418 return length;
9419}
9420SLAB_ATTR(failslab);
9421#endif
9422
9423static ssize_t shrink_show(struct kmem_cache *s, char *buf)
9424{
9425 return 0;
9426}
9427
9428static ssize_t shrink_store(struct kmem_cache *s,
9429 const char *buf, size_t length)
9430{
9431 if (buf[0] == '1')
9432 kmem_cache_shrink(s);
9433 else
9434 return -EINVAL;
9435 return length;
9436}
9437SLAB_ATTR(shrink);
9438
9439#ifdef CONFIG_NUMA
9440static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
9441{
9442 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
9443}
9444
9445static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
9446 const char *buf, size_t length)
9447{
9448 unsigned int ratio;
9449 int err;
9450
9451 err = kstrtouint(buf, 10, &ratio);
9452 if (err)
9453 return err;
9454 if (ratio > 100)
9455 return -ERANGE;
9456
9457 s->remote_node_defrag_ratio = ratio * 10;
9458
9459 return length;
9460}
9461SLAB_ATTR(remote_node_defrag_ratio);
9462#endif
9463
9464#ifdef CONFIG_SLUB_STATS
9465static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
9466{
9467 unsigned long sum = 0;
9468 int cpu;
9469 int len = 0;
9470 int *data = kmalloc_objs(int, nr_cpu_ids);
9471
9472 if (!data)
9473 return -ENOMEM;
9474
9475 for_each_online_cpu(cpu) {
9476 unsigned int x = per_cpu_ptr(s->cpu_stats, cpu)->stat[si];
9477
9478 data[cpu] = x;
9479 sum += x;
9480 }
9481
9482 len += sysfs_emit_at(buf, len, "%lu", sum);
9483
9484#ifdef CONFIG_SMP
9485 for_each_online_cpu(cpu) {
9486 if (data[cpu])
9487 len += sysfs_emit_at(buf, len, " C%d=%u",
9488 cpu, data[cpu]);
9489 }
9490#endif
9491 kfree(data);
9492 len += sysfs_emit_at(buf, len, "\n");
9493
9494 return len;
9495}
9496
9497static void clear_stat(struct kmem_cache *s, enum stat_item si)
9498{
9499 int cpu;
9500
9501 for_each_online_cpu(cpu)
9502 per_cpu_ptr(s->cpu_stats, cpu)->stat[si] = 0;
9503}
9504
9505#define STAT_ATTR(si, text) \
9506static ssize_t text##_show(struct kmem_cache *s, char *buf) \
9507{ \
9508 return show_stat(s, buf, si); \
9509} \
9510static ssize_t text##_store(struct kmem_cache *s, \
9511 const char *buf, size_t length) \
9512{ \
9513 if (buf[0] != '0') \
9514 return -EINVAL; \
9515 clear_stat(s, si); \
9516 return length; \
9517} \
9518SLAB_ATTR(text); \
9519
9520STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
9521STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
9522STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf);
9523STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail);
9524STAT_ATTR(FREE_FASTPATH, free_fastpath);
9525STAT_ATTR(FREE_SLOWPATH, free_slowpath);
9526STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
9527STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
9528STAT_ATTR(ALLOC_SLAB, alloc_slab);
9529STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
9530STAT_ATTR(FREE_SLAB, free_slab);
9531STAT_ATTR(ORDER_FALLBACK, order_fallback);
9532STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
9533STAT_ATTR(SHEAF_FLUSH, sheaf_flush);
9534STAT_ATTR(SHEAF_REFILL, sheaf_refill);
9535STAT_ATTR(SHEAF_ALLOC, sheaf_alloc);
9536STAT_ATTR(SHEAF_FREE, sheaf_free);
9537STAT_ATTR(BARN_GET, barn_get);
9538STAT_ATTR(BARN_GET_FAIL, barn_get_fail);
9539STAT_ATTR(BARN_PUT, barn_put);
9540STAT_ATTR(BARN_PUT_FAIL, barn_put_fail);
9541STAT_ATTR(SHEAF_PREFILL_FAST, sheaf_prefill_fast);
9542STAT_ATTR(SHEAF_PREFILL_SLOW, sheaf_prefill_slow);
9543STAT_ATTR(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize);
9544STAT_ATTR(SHEAF_RETURN_FAST, sheaf_return_fast);
9545STAT_ATTR(SHEAF_RETURN_SLOW, sheaf_return_slow);
9546#endif /* CONFIG_SLUB_STATS */
9547
9548#ifdef CONFIG_KFENCE
9549static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)
9550{
9551 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE));
9552}
9553
9554static ssize_t skip_kfence_store(struct kmem_cache *s,
9555 const char *buf, size_t length)
9556{
9557 int ret = length;
9558
9559 if (buf[0] == '0')
9560 s->flags &= ~SLAB_SKIP_KFENCE;
9561 else if (buf[0] == '1')
9562 s->flags |= SLAB_SKIP_KFENCE;
9563 else
9564 ret = -EINVAL;
9565
9566 return ret;
9567}
9568SLAB_ATTR(skip_kfence);
9569#endif
9570
9571static const struct attribute *const slab_attrs[] = {
9572 &slab_size_attr.attr,
9573 &object_size_attr.attr,
9574 &objs_per_slab_attr.attr,
9575 &order_attr.attr,
9576 &sheaf_capacity_attr.attr,
9577 &min_partial_attr.attr,
9578 &cpu_partial_attr.attr,
9579 &objects_partial_attr.attr,
9580 &partial_attr.attr,
9581 &cpu_slabs_attr.attr,
9582 &ctor_attr.attr,
9583 &aliases_attr.attr,
9584 &align_attr.attr,
9585 &hwcache_align_attr.attr,
9586 &reclaim_account_attr.attr,
9587 &destroy_by_rcu_attr.attr,
9588 &shrink_attr.attr,
9589 &slabs_cpu_partial_attr.attr,
9590#ifdef CONFIG_SLUB_DEBUG
9591 &total_objects_attr.attr,
9592 &objects_attr.attr,
9593 &slabs_attr.attr,
9594 &sanity_checks_attr.attr,
9595 &trace_attr.attr,
9596 &red_zone_attr.attr,
9597 &poison_attr.attr,
9598 &store_user_attr.attr,
9599 &validate_attr.attr,
9600#endif
9601#ifdef CONFIG_ZONE_DMA
9602 &cache_dma_attr.attr,
9603#endif
9604#ifdef CONFIG_NUMA
9605 &remote_node_defrag_ratio_attr.attr,
9606#endif
9607#ifdef CONFIG_SLUB_STATS
9608 &alloc_fastpath_attr.attr,
9609 &alloc_slowpath_attr.attr,
9610 &free_rcu_sheaf_attr.attr,
9611 &free_rcu_sheaf_fail_attr.attr,
9612 &free_fastpath_attr.attr,
9613 &free_slowpath_attr.attr,
9614 &free_add_partial_attr.attr,
9615 &free_remove_partial_attr.attr,
9616 &alloc_slab_attr.attr,
9617 &alloc_node_mismatch_attr.attr,
9618 &free_slab_attr.attr,
9619 &order_fallback_attr.attr,
9620 &cmpxchg_double_fail_attr.attr,
9621 &sheaf_flush_attr.attr,
9622 &sheaf_refill_attr.attr,
9623 &sheaf_alloc_attr.attr,
9624 &sheaf_free_attr.attr,
9625 &barn_get_attr.attr,
9626 &barn_get_fail_attr.attr,
9627 &barn_put_attr.attr,
9628 &barn_put_fail_attr.attr,
9629 &sheaf_prefill_fast_attr.attr,
9630 &sheaf_prefill_slow_attr.attr,
9631 &sheaf_prefill_oversize_attr.attr,
9632 &sheaf_return_fast_attr.attr,
9633 &sheaf_return_slow_attr.attr,
9634#endif
9635#ifdef CONFIG_FAILSLAB
9636 &failslab_attr.attr,
9637#endif
9638#ifdef CONFIG_HARDENED_USERCOPY
9639 &usersize_attr.attr,
9640#endif
9641#ifdef CONFIG_KFENCE
9642 &skip_kfence_attr.attr,
9643#endif
9644
9645 NULL
9646};
9647
9648ATTRIBUTE_GROUPS(slab);
9649
9650static ssize_t slab_attr_show(struct kobject *kobj,
9651 struct attribute *attr,
9652 char *buf)
9653{
9654 const struct slab_attribute *attribute;
9655 struct kmem_cache *s;
9656
9657 attribute = to_slab_attr(attr);
9658 s = to_slab(kobj);
9659
9660 if (!attribute->show)
9661 return -EIO;
9662
9663 return attribute->show(s, buf);
9664}
9665
9666static ssize_t slab_attr_store(struct kobject *kobj,
9667 struct attribute *attr,
9668 const char *buf, size_t len)
9669{
9670 const struct slab_attribute *attribute;
9671 struct kmem_cache *s;
9672
9673 attribute = to_slab_attr(attr);
9674 s = to_slab(kobj);
9675
9676 if (!attribute->store)
9677 return -EIO;
9678
9679 return attribute->store(s, buf, len);
9680}
9681
9682static void kmem_cache_release(struct kobject *k)
9683{
9684 slab_kmem_cache_release(to_slab(k));
9685}
9686
9687static const struct sysfs_ops slab_sysfs_ops = {
9688 .show = slab_attr_show,
9689 .store = slab_attr_store,
9690};
9691
9692static const struct kobj_type slab_ktype = {
9693 .sysfs_ops = &slab_sysfs_ops,
9694 .release = kmem_cache_release,
9695 .default_groups = slab_groups,
9696};
9697
9698static struct kset *slab_kset;
9699
9700static inline struct kset *cache_kset(struct kmem_cache *s)
9701{
9702 return slab_kset;
9703}
9704
9705#define ID_STR_LENGTH 32
9706
9707/* Create a unique string id for a slab cache:
9708 *
9709 * Format :[flags-]size
9710 */
9711static char *create_unique_id(struct kmem_cache *s)
9712{
9713 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
9714 char *p = name;
9715
9716 if (!name)
9717 return ERR_PTR(-ENOMEM);
9718
9719 *p++ = ':';
9720 /*
9721 * First flags affecting slabcache operations. We will only
9722 * get here for aliasable slabs so we do not need to support
9723 * too many flags. The flags here must cover all flags that
9724 * are matched during merging to guarantee that the id is
9725 * unique.
9726 */
9727 if (s->flags & SLAB_CACHE_DMA)
9728 *p++ = 'd';
9729 if (s->flags & SLAB_CACHE_DMA32)
9730 *p++ = 'D';
9731 if (s->flags & SLAB_RECLAIM_ACCOUNT)
9732 *p++ = 'a';
9733 if (s->flags & SLAB_CONSISTENCY_CHECKS)
9734 *p++ = 'F';
9735 if (s->flags & SLAB_ACCOUNT)
9736 *p++ = 'A';
9737 if (p != name + 1)
9738 *p++ = '-';
9739 p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size);
9740
9741 if (WARN_ON(p > name + ID_STR_LENGTH - 1)) {
9742 kfree(name);
9743 return ERR_PTR(-EINVAL);
9744 }
9745 kmsan_unpoison_memory(name, p - name);
9746 return name;
9747}
9748
9749static int sysfs_slab_add(struct kmem_cache *s)
9750{
9751 int err;
9752 const char *name;
9753 struct kset *kset = cache_kset(s);
9754 int unmergeable = slab_unmergeable(s);
9755
9756 if (!unmergeable && disable_higher_order_debug &&
9757 (slub_debug & DEBUG_METADATA_FLAGS))
9758 unmergeable = 1;
9759
9760 if (unmergeable) {
9761 /*
9762 * Slabcache can never be merged so we can use the name proper.
9763 * This is typically the case for debug situations. In that
9764 * case we can catch duplicate names easily.
9765 */
9766 sysfs_remove_link(&slab_kset->kobj, s->name);
9767 name = s->name;
9768 } else {
9769 /*
9770 * Create a unique name for the slab as a target
9771 * for the symlinks.
9772 */
9773 name = create_unique_id(s);
9774 if (IS_ERR(name))
9775 return PTR_ERR(name);
9776 }
9777
9778 s->kobj.kset = kset;
9779 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
9780 /*
9781 * Intentionally skip kobject_put(). See commit 2420baa8e046
9782 * ("mm/slab: Allow cache creation to proceed even if sysfs
9783 * registration fails")
9784 */
9785 if (err)
9786 goto out;
9787
9788 if (!unmergeable) {
9789 /* Setup first alias */
9790 sysfs_slab_alias(s, s->name);
9791 }
9792out:
9793 if (!unmergeable)
9794 kfree(name);
9795 return err;
9796}
9797
9798void sysfs_slab_unlink(struct kmem_cache *s)
9799{
9800 if (s->kobj.state_in_sysfs)
9801 kobject_del(&s->kobj);
9802}
9803
9804void sysfs_slab_release(struct kmem_cache *s)
9805{
9806 kobject_put(&s->kobj);
9807}
9808
9809/*
9810 * Need to buffer aliases during bootup until sysfs becomes
9811 * available lest we lose that information.
9812 */
9813struct saved_alias {
9814 struct kmem_cache *s;
9815 const char *name;
9816 struct saved_alias *next;
9817};
9818
9819static struct saved_alias *alias_list;
9820
9821int sysfs_slab_alias(struct kmem_cache *s, const char *name)
9822{
9823 struct saved_alias *al;
9824
9825 if (slab_state == FULL) {
9826 /*
9827 * If we have a leftover link then remove it.
9828 */
9829 sysfs_remove_link(&slab_kset->kobj, name);
9830 /*
9831 * The original cache may have failed to generate sysfs file.
9832 * In that case, sysfs_create_link() returns -ENOENT and
9833 * symbolic link creation is skipped.
9834 */
9835 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
9836 }
9837
9838 al = kmalloc_obj(struct saved_alias);
9839 if (!al)
9840 return -ENOMEM;
9841
9842 al->s = s;
9843 al->name = name;
9844 al->next = alias_list;
9845 alias_list = al;
9846 kmsan_unpoison_memory(al, sizeof(*al));
9847 return 0;
9848}
9849
9850static int __init slab_kset_init(void)
9851{
9852 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
9853 if (!slab_kset) {
9854 pr_err("Cannot register slab subsystem.\n");
9855 return -ENOMEM;
9856 }
9857
9858 return 0;
9859}
9860
9861static void __init slab_sysfs_process_aliases(void)
9862{
9863 int err;
9864
9865 while (alias_list) {
9866 struct saved_alias *al = alias_list;
9867
9868 alias_list = alias_list->next;
9869 err = sysfs_slab_alias(al->s, al->name);
9870 if (err)
9871 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
9872 al->name);
9873 kfree(al);
9874 }
9875}
9876#endif /* SLAB_SUPPORTS_SYSFS */
9877
9878#if defined(SLAB_SUPPORTS_SYSFS) || \
9879 (defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS))
9880static int __init slab_late_init(void)
9881{
9882 struct kmem_cache *s;
9883 int err;
9884
9885 mutex_lock(&slab_mutex);
9886
9887 err = slab_kset_init();
9888 if (err)
9889 goto out;
9890
9891 slab_debugfs_root_init();
9892 slab_state = FULL;
9893
9894 list_for_each_entry(s, &slab_caches, list) {
9895 if (sysfs_slab_add(s))
9896 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
9897 s->name);
9898
9899 if (s->flags & SLAB_STORE_USER)
9900 debugfs_slab_add(s);
9901 }
9902
9903 slab_sysfs_process_aliases();
9904out:
9905 mutex_unlock(&slab_mutex);
9906 return err;
9907}
9908late_initcall(slab_late_init);
9909#endif
9910
9911#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
9912static int slab_debugfs_show(struct seq_file *seq, void *v)
9913{
9914 struct loc_track *t = seq->private;
9915 struct location *l;
9916 unsigned long idx;
9917
9918 idx = (unsigned long) t->idx;
9919 if (idx < t->count) {
9920 l = &t->loc[idx];
9921
9922 seq_printf(seq, "%7ld ", l->count);
9923
9924 if (l->addr)
9925 seq_printf(seq, "%pS", (void *)l->addr);
9926 else
9927 seq_puts(seq, "<not-available>");
9928
9929 if (l->waste)
9930 seq_printf(seq, " waste=%lu/%lu",
9931 l->count * l->waste, l->waste);
9932
9933 if (l->sum_time != l->min_time) {
9934 seq_printf(seq, " age=%ld/%llu/%ld",
9935 l->min_time, div_u64(l->sum_time, l->count),
9936 l->max_time);
9937 } else
9938 seq_printf(seq, " age=%ld", l->min_time);
9939
9940 if (l->min_pid != l->max_pid)
9941 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
9942 else
9943 seq_printf(seq, " pid=%ld",
9944 l->min_pid);
9945
9946 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
9947 seq_printf(seq, " cpus=%*pbl",
9948 cpumask_pr_args(to_cpumask(l->cpus)));
9949
9950 if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
9951 seq_printf(seq, " nodes=%*pbl",
9952 nodemask_pr_args(&l->nodes));
9953
9954#ifdef CONFIG_STACKDEPOT
9955 {
9956 depot_stack_handle_t handle;
9957 unsigned long *entries;
9958 unsigned int nr_entries, j;
9959
9960 handle = READ_ONCE(l->handle);
9961 if (handle) {
9962 nr_entries = stack_depot_fetch(handle, &entries);
9963 seq_puts(seq, "\n");
9964 for (j = 0; j < nr_entries; j++)
9965 seq_printf(seq, " %pS\n", (void *)entries[j]);
9966 }
9967 }
9968#endif
9969 seq_puts(seq, "\n");
9970 }
9971
9972 if (!idx && !t->count)
9973 seq_puts(seq, "No data\n");
9974
9975 return 0;
9976}
9977
9978static void slab_debugfs_stop(struct seq_file *seq, void *v)
9979{
9980}
9981
9982static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
9983{
9984 struct loc_track *t = seq->private;
9985
9986 t->idx = ++(*ppos);
9987 if (*ppos <= t->count)
9988 return ppos;
9989
9990 return NULL;
9991}
9992
9993static int cmp_loc_by_count(const void *a, const void *b)
9994{
9995 struct location *loc1 = (struct location *)a;
9996 struct location *loc2 = (struct location *)b;
9997
9998 return cmp_int(loc2->count, loc1->count);
9999}
10000
10001static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
10002{
10003 struct loc_track *t = seq->private;
10004
10005 t->idx = *ppos;
10006 return ppos;
10007}
10008
10009static const struct seq_operations slab_debugfs_sops = {
10010 .start = slab_debugfs_start,
10011 .next = slab_debugfs_next,
10012 .stop = slab_debugfs_stop,
10013 .show = slab_debugfs_show,
10014};
10015
10016static int slab_debug_trace_open(struct inode *inode, struct file *filep)
10017{
10018
10019 struct kmem_cache_node *n;
10020 enum track_item alloc;
10021 int node;
10022 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
10023 sizeof(struct loc_track));
10024 struct kmem_cache *s = file_inode(filep)->i_private;
10025 unsigned long *obj_map;
10026
10027 if (!t)
10028 return -ENOMEM;
10029
10030 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
10031 if (!obj_map) {
10032 seq_release_private(inode, filep);
10033 return -ENOMEM;
10034 }
10035
10036 alloc = debugfs_get_aux_num(filep);
10037
10038 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
10039 bitmap_free(obj_map);
10040 seq_release_private(inode, filep);
10041 return -ENOMEM;
10042 }
10043
10044 for_each_kmem_cache_node(s, node, n) {
10045 unsigned long flags;
10046 struct slab *slab;
10047
10048 if (!node_nr_slabs(n))
10049 continue;
10050
10051 spin_lock_irqsave(&n->list_lock, flags);
10052 list_for_each_entry(slab, &n->partial, slab_list)
10053 process_slab(t, s, slab, alloc, obj_map);
10054 list_for_each_entry(slab, &n->full, slab_list)
10055 process_slab(t, s, slab, alloc, obj_map);
10056 spin_unlock_irqrestore(&n->list_lock, flags);
10057 }
10058
10059 /* Sort locations by count */
10060 sort(t->loc, t->count, sizeof(struct location),
10061 cmp_loc_by_count, NULL);
10062
10063 bitmap_free(obj_map);
10064 return 0;
10065}
10066
10067static int slab_debug_trace_release(struct inode *inode, struct file *file)
10068{
10069 struct seq_file *seq = file->private_data;
10070 struct loc_track *t = seq->private;
10071
10072 free_loc_track(t);
10073 return seq_release_private(inode, file);
10074}
10075
10076static const struct file_operations slab_debugfs_fops = {
10077 .open = slab_debug_trace_open,
10078 .read = seq_read,
10079 .llseek = seq_lseek,
10080 .release = slab_debug_trace_release,
10081};
10082
10083static void debugfs_slab_add(struct kmem_cache *s)
10084{
10085 struct dentry *slab_cache_dir;
10086
10087 if (unlikely(!slab_debugfs_root))
10088 return;
10089
10090 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
10091
10092 debugfs_create_file_aux_num("alloc_traces", 0400, slab_cache_dir, s,
10093 TRACK_ALLOC, &slab_debugfs_fops);
10094
10095 debugfs_create_file_aux_num("free_traces", 0400, slab_cache_dir, s,
10096 TRACK_FREE, &slab_debugfs_fops);
10097}
10098
10099void debugfs_slab_release(struct kmem_cache *s)
10100{
10101 if (unlikely(!slab_debugfs_root))
10102 return;
10103
10104 debugfs_lookup_and_remove(s->name, slab_debugfs_root);
10105}
10106
10107static void __init slab_debugfs_root_init(void)
10108{
10109 slab_debugfs_root = debugfs_create_dir("slab", NULL);
10110}
10111#endif
10112/*
10113 * The /proc/slabinfo ABI
10114 */
10115#ifdef CONFIG_SLUB_DEBUG
10116void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
10117{
10118 unsigned long nr_slabs = 0;
10119 unsigned long nr_objs = 0;
10120 unsigned long nr_free = 0;
10121 int node;
10122 struct kmem_cache_node *n;
10123
10124 for_each_kmem_cache_node(s, node, n) {
10125 nr_slabs += node_nr_slabs(n);
10126 nr_objs += node_nr_objs(n);
10127 nr_free += count_partial_free_approx(n);
10128 }
10129
10130 sinfo->active_objs = nr_objs - nr_free;
10131 sinfo->num_objs = nr_objs;
10132 sinfo->active_slabs = nr_slabs;
10133 sinfo->num_slabs = nr_slabs;
10134 sinfo->objects_per_slab = oo_objects(s->oo);
10135 sinfo->cache_order = oo_order(s->oo);
10136}
10137#endif /* CONFIG_SLUB_DEBUG */