projects/linux·blob master · refs

962 lines · 25.6 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef MM_SLAB_H
3#define MM_SLAB_H
4
5#include <linux/reciprocal_div.h>
6#include <linux/list_lru.h>
7#include <linux/local_lock.h>
8#include <linux/random.h>
9#include <linux/kobject.h>
10#include <linux/sched/mm.h>
11#include <linux/memcontrol.h>
12#include <linux/kfence.h>
13#include <linux/kasan.h>
14#include <linux/slab.h>
15
16/*
17 * Internal slab definitions
18 */
19
20/* slab's alloc_flags definitions */
21#define SLAB_ALLOC_DEFAULT 0x00 /* no flags */
22#define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */
23#define SLAB_ALLOC_NEW_SLAB 0x02 /* a flag for alloc_slab_obj_exts() */
24#define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */
25#define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */
26
27#define SLAB_FREE_DEFAULT 0x00 /* no flags */
28#define SLAB_FREE_NOLOCK 0x01 /* spinning not allowed */
29
30static inline unsigned int to_alloc_flags(unsigned int free_flags)
31{
32 if (free_flags & SLAB_FREE_NOLOCK)
33 return SLAB_ALLOC_NOLOCK;
34 else
35 return SLAB_ALLOC_DEFAULT;
36}
37
38static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
39{
40 return !(alloc_flags & SLAB_ALLOC_NOLOCK);
41}
42
43static inline bool free_flags_allow_spinning(const unsigned int free_flags)
44{
45 return !(free_flags & SLAB_FREE_NOLOCK);
46}
47
48void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags,
49 unsigned int alloc_flags, int node)
50 __assume_kmalloc_alignment __alloc_size(1);
51
52static __always_inline __alloc_size(1) void *_kmalloc_flags_noprof(size_t size,
53 gfp_t flags, unsigned int alloc_flags, int node, kmalloc_token_t token)
54{
55 return __kmalloc_flags_noprof(PASS_TOKEN_PARAMS(size, token), flags, alloc_flags, node);
56}
57#define kmalloc_flags_noprof(...) _kmalloc_flags_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
58#define kmalloc_flags(...) alloc_hooks(kmalloc_flags_noprof(__VA_ARGS__))
59
60#ifdef CONFIG_64BIT
61# ifdef system_has_cmpxchg128
62# define system_has_freelist_aba() system_has_cmpxchg128()
63# define try_cmpxchg_freelist try_cmpxchg128
64# endif
65typedef u128 freelist_full_t;
66#else /* CONFIG_64BIT */
67# ifdef system_has_cmpxchg64
68# define system_has_freelist_aba() system_has_cmpxchg64()
69# define try_cmpxchg_freelist try_cmpxchg64
70# endif
71typedef u64 freelist_full_t;
72#endif /* CONFIG_64BIT */
73
74#if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
75#undef system_has_freelist_aba
76#endif
77
78/*
79 * Freelist pointer and counter to cmpxchg together, avoids the typical ABA
80 * problems with cmpxchg of just a pointer.
81 */
82struct freelist_counters {
83 union {
84 struct {
85 void *freelist;
86 union {
87 unsigned long counters;
88 struct {
89 unsigned inuse:16;
90 unsigned objects:15;
91 /*
92 * If slab debugging is enabled then the
93 * frozen bit can be reused to indicate
94 * that the slab was corrupted
95 */
96 unsigned frozen:1;
97#ifdef CONFIG_64BIT
98 /*
99 * Some optimizations use free bits in 'counters' field
100 * to save memory or CPU. If these free bits are not
101 * available, such optimizations are disabled.
102 */
103 unsigned obj_exts_in_object:1;
104 unsigned obj_exts_needs_objcg:1;
105#endif
106 };
107 };
108 };
109#ifdef system_has_freelist_aba
110 freelist_full_t freelist_counters;
111#endif
112 };
113};
114
115/* Reuses the bits in struct page */
116struct slab {
117 memdesc_flags_t flags;
118
119 struct kmem_cache *slab_cache;
120 union {
121 struct {
122 struct list_head slab_list;
123 /* Double-word boundary */
124 struct freelist_counters;
125 };
126 struct rcu_head rcu_head;
127 };
128
129 unsigned int __page_type;
130 atomic_t __page_refcount;
131#ifdef CONFIG_SLAB_OBJ_EXT
132 unsigned long obj_exts;
133#endif
134};
135
136#define SLAB_MATCH(pg, sl) \
137 static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))
138SLAB_MATCH(flags, flags);
139SLAB_MATCH(compound_info, slab_cache); /* Ensure bit 0 is clear */
140SLAB_MATCH(_refcount, __page_refcount);
141#ifdef CONFIG_MEMCG
142SLAB_MATCH(memcg_data, obj_exts);
143#elif defined(CONFIG_SLAB_OBJ_EXT)
144SLAB_MATCH(_unused_slab_obj_exts, obj_exts);
145#endif
146#undef SLAB_MATCH
147static_assert(sizeof(struct slab) <= sizeof(struct page));
148#if defined(system_has_freelist_aba)
149static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist_counters)));
150#endif
151
152/**
153 * slab_folio - The folio allocated for a slab
154 * @s: The slab.
155 *
156 * Slabs are allocated as folios that contain the individual objects and are
157 * using some fields in the first struct page of the folio - those fields are
158 * now accessed by struct slab. It is occasionally necessary to convert back to
159 * a folio in order to communicate with the rest of the mm. Please use this
160 * helper function instead of casting yourself, as the implementation may change
161 * in the future.
162 */
163#define slab_folio(s) (_Generic((s), \
164 const struct slab *: (const struct folio *)s, \
165 struct slab *: (struct folio *)s))
166
167/**
168 * page_slab - Converts from struct page to its slab.
169 * @page: A page which may or may not belong to a slab.
170 *
171 * Return: The slab which contains this page or NULL if the page does
172 * not belong to a slab. This includes pages returned from large kmalloc.
173 */
174static inline struct slab *page_slab(const struct page *page)
175{
176 page = compound_head(page);
177 if (data_race(page->page_type >> 24) != PGTY_slab)
178 page = NULL;
179
180 return (struct slab *)page;
181}
182
183/**
184 * slab_page - The first struct page allocated for a slab
185 * @s: The slab.
186 *
187 * A convenience wrapper for converting slab to the first struct page of the
188 * underlying folio, to communicate with code not yet converted to folio or
189 * struct slab.
190 */
191#define slab_page(s) folio_page(slab_folio(s), 0)
192
193static inline void *slab_address(const struct slab *slab)
194{
195 return folio_address(slab_folio(slab));
196}
197
198static inline int slab_nid(const struct slab *slab)
199{
200 return memdesc_nid(&slab->flags);
201}
202
203static inline pg_data_t *slab_pgdat(const struct slab *slab)
204{
205 return NODE_DATA(slab_nid(slab));
206}
207
208static inline struct slab *virt_to_slab(const void *addr)
209{
210 return page_slab(virt_to_page(addr));
211}
212
213static inline int slab_order(const struct slab *slab)
214{
215 return folio_order(slab_folio(slab));
216}
217
218static inline size_t slab_size(const struct slab *slab)
219{
220 return PAGE_SIZE << slab_order(slab);
221}
222
223/*
224 * Word size structure that can be atomically updated or read and that
225 * contains both the order and the number of objects that a slab of the
226 * given order would contain.
227 */
228struct kmem_cache_order_objects {
229 unsigned int x;
230};
231
232struct kmem_cache_per_node_ptrs {
233 struct node_barn *barn;
234 struct kmem_cache_node *node;
235};
236
237/*
238 * Slab cache management.
239 */
240struct kmem_cache {
241 struct slub_percpu_sheaves __percpu *cpu_sheaves;
242 /* Used for retrieving partial slabs, etc. */
243 slab_flags_t flags;
244 unsigned long min_partial;
245 unsigned int size; /* Object size including metadata */
246 unsigned int object_size; /* Object size without metadata */
247 struct reciprocal_value reciprocal_size;
248 unsigned int offset; /* Free pointer offset */
249 unsigned int sheaf_capacity;
250 struct kmem_cache_order_objects oo;
251
252 /* Allocation and freeing of slabs */
253 struct kmem_cache_order_objects min;
254 gfp_t allocflags; /* gfp flags to use on each alloc */
255 int refcount; /* Refcount for slab cache destroy */
256 void (*ctor)(void *object); /* Object constructor */
257 unsigned int inuse; /* Offset to metadata */
258 unsigned int align; /* Alignment */
259 unsigned int red_left_pad; /* Left redzone padding size */
260 const char *name; /* Name (only for display!) */
261 struct list_head list; /* List of slab caches */
262#ifdef CONFIG_SYSFS
263 struct kobject kobj; /* For sysfs */
264#endif
265#ifdef CONFIG_SLAB_FREELIST_HARDENED
266 unsigned long random;
267#endif
268
269#ifdef CONFIG_NUMA
270 /*
271 * Defragmentation by allocating from a remote node.
272 */
273 unsigned int remote_node_defrag_ratio;
274#endif
275
276#ifdef CONFIG_SLAB_FREELIST_RANDOM
277 unsigned int *random_seq;
278#endif
279
280#ifdef CONFIG_KASAN_GENERIC
281 struct kasan_cache kasan_info;
282#endif
283
284#ifdef CONFIG_HARDENED_USERCOPY
285 unsigned int useroffset; /* Usercopy region offset */
286 unsigned int usersize; /* Usercopy region size */
287#endif
288
289#ifdef CONFIG_SLUB_STATS
290 struct kmem_cache_stats __percpu *cpu_stats;
291#endif
292
293 struct kmem_cache_per_node_ptrs per_node[MAX_NUMNODES];
294};
295
296/*
297 * Every cache has !NULL s->cpu_sheaves but they may point to the
298 * bootstrap_sheaf temporarily during init, or permanently for the boot caches
299 * and caches with debugging enabled, or all caches with CONFIG_SLUB_TINY. This
300 * helper distinguishes whether cache has real non-bootstrap sheaves.
301 */
302static inline bool cache_has_sheaves(struct kmem_cache *s)
303{
304 /* Test CONFIG_SLUB_TINY for code elimination purposes */
305 return !IS_ENABLED(CONFIG_SLUB_TINY) && s->sheaf_capacity;
306}
307
308#if defined(CONFIG_SYSFS) && !defined(CONFIG_SLUB_TINY)
309#define SLAB_SUPPORTS_SYSFS 1
310void sysfs_slab_unlink(struct kmem_cache *s);
311void sysfs_slab_release(struct kmem_cache *s);
312int sysfs_slab_alias(struct kmem_cache *s, const char *name);
313#else
314static inline void sysfs_slab_unlink(struct kmem_cache *s) { }
315static inline void sysfs_slab_release(struct kmem_cache *s) { }
316static inline int sysfs_slab_alias(struct kmem_cache *s, const char *name)
317 { return 0; }
318#endif
319
320void *fixup_red_left(struct kmem_cache *s, void *p);
321
322static inline void *nearest_obj(struct kmem_cache *cache,
323 const struct slab *slab, void *x)
324{
325 void *object = x - (x - slab_address(slab)) % cache->size;
326 void *last_object = slab_address(slab) +
327 (slab->objects - 1) * cache->size;
328 void *result = (unlikely(object > last_object)) ? last_object : object;
329
330 result = fixup_red_left(cache, result);
331 return result;
332}
333
334/* Determine object index from a given position */
335static inline unsigned int __obj_to_index(const struct kmem_cache *cache,
336 void *addr, const void *obj)
337{
338 return reciprocal_divide(kasan_reset_tag(obj) - addr,
339 cache->reciprocal_size);
340}
341
342static inline unsigned int obj_to_index(const struct kmem_cache *cache,
343 const struct slab *slab, const void *obj)
344{
345 if (is_kfence_address(obj))
346 return 0;
347 return __obj_to_index(cache, slab_address(slab), obj);
348}
349
350/*
351 * kvfree_rcu_head offset can be only less than page size.
352 * Calculate the start address while preserving the KASAN tag.
353 */
354static inline void *kvmalloc_obj_start_addr(void *head)
355{
356 unsigned long offset;
357
358 if (unlikely(is_vmalloc_addr(head))) {
359 offset = offset_in_page(head);
360 } else {
361 struct slab *slab = virt_to_slab(head);
362
363 if (!slab) {
364 offset = offset_in_page(head);
365 } else if (is_kfence_address(head)) {
366 offset = head - kfence_object_start(head);
367 } else {
368 struct kmem_cache *s = slab->slab_cache;
369 unsigned int idx = __obj_to_index(s, slab_address(slab), head);
370 void *obj = slab_address(slab) + s->size * idx;
371
372 obj = fixup_red_left(s, obj);
373 obj = kasan_reset_tag(obj);
374 offset = kasan_reset_tag(head) - obj;
375 }
376 }
377
378 return head - offset;
379}
380
381/*
382 * State of the slab allocator.
383 *
384 * This is used to describe the states of the allocator during bootup.
385 * Allocators use this to gradually bootstrap themselves. Most allocators
386 * have the problem that the structures used for managing slab caches are
387 * allocated from slab caches themselves.
388 */
389enum slab_state {
390 DOWN, /* No slab functionality yet */
391 PARTIAL, /* SLUB: kmem_cache_node available */
392 UP, /* Slab caches usable but not all extras yet */
393 FULL /* Everything is working */
394};
395
396extern enum slab_state slab_state;
397
398/* The slab cache mutex protects the management structures during changes */
399extern struct mutex slab_mutex;
400
401/* The list of all slab caches on the system */
402extern struct list_head slab_caches;
403
404/* The slab cache that manages slab cache information */
405extern struct kmem_cache *kmem_cache;
406
407/* A table of kmalloc cache names and sizes */
408extern const struct kmalloc_info_struct {
409 const char *name[NR_KMALLOC_TYPES];
410 unsigned int size;
411} kmalloc_info[];
412
413/* Kmalloc array related functions */
414void setup_kmalloc_cache_index_table(void);
415void create_kmalloc_caches(void);
416
417extern u8 kmalloc_size_index[24];
418
419static inline unsigned int size_index_elem(unsigned int bytes)
420{
421 return (bytes - 1) / 8;
422}
423
424/*
425 * Find the kmem_cache structure that serves a given size of
426 * allocation
427 *
428 * This assumes size is larger than zero and not larger than
429 * KMALLOC_MAX_CACHE_SIZE and the caller must check that.
430 */
431static inline struct kmem_cache *
432kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token,
433 unsigned int alloc_flags)
434{
435 unsigned int index;
436 enum kmalloc_cache_type type = kmalloc_type(flags, token);
437
438 if (alloc_flags & SLAB_ALLOC_NO_OBJ_EXT)
439 type = KMALLOC_NO_OBJ_EXT;
440
441 if (!b)
442 b = &kmalloc_caches[type];
443 if (size <= 192)
444 index = kmalloc_size_index[size_index_elem(size)];
445 else
446 index = fls(size - 1);
447
448 return (*b)[index];
449}
450
451gfp_t kmalloc_fix_flags(gfp_t flags);
452
453/* Functions provided by the slab allocators */
454int do_kmem_cache_create(struct kmem_cache *s, const char *name,
455 unsigned int size, struct kmem_cache_args *args,
456 slab_flags_t flags);
457
458void __init kmem_cache_init(void);
459extern void create_boot_cache(struct kmem_cache *, const char *name,
460 unsigned int size, slab_flags_t flags,
461 unsigned int useroffset, unsigned int usersize);
462
463int slab_unmergeable(struct kmem_cache *s);
464bool slab_args_unmergeable(struct kmem_cache_args *args, slab_flags_t flags);
465
466slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name);
467
468static inline bool is_kmalloc_cache(struct kmem_cache *s)
469{
470 return (s->flags & SLAB_KMALLOC);
471}
472
473static inline bool is_kmalloc_normal(struct kmem_cache *s)
474{
475 if (!is_kmalloc_cache(s))
476 return false;
477
478 return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT));
479}
480
481bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags);
482void flush_all_rcu_sheaves(void);
483void flush_rcu_sheaves_on_cache(struct kmem_cache *s);
484
485#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
486 SLAB_CACHE_DMA32 | SLAB_PANIC | \
487 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS | \
488 SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
489 SLAB_TEMPORARY | SLAB_ACCOUNT | \
490 SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE)
491
492#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
493 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
494
495#define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS)
496
497bool __kmem_cache_empty(struct kmem_cache *);
498int __kmem_cache_shutdown(struct kmem_cache *);
499void __kmem_cache_release(struct kmem_cache *);
500int __kmem_cache_shrink(struct kmem_cache *);
501void slab_kmem_cache_release(struct kmem_cache *);
502
503struct seq_file;
504struct file;
505
506struct slabinfo {
507 unsigned long active_objs;
508 unsigned long num_objs;
509 unsigned long active_slabs;
510 unsigned long num_slabs;
511 unsigned long shared_avail;
512 unsigned int limit;
513 unsigned int batchcount;
514 unsigned int shared;
515 unsigned int objects_per_slab;
516 unsigned int cache_order;
517};
518
519void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
520
521#ifdef CONFIG_SLUB_DEBUG
522#ifdef CONFIG_SLUB_DEBUG_ON
523DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
524#else
525DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
526#endif
527extern void print_tracking(struct kmem_cache *s, void *object);
528long validate_slab_cache(struct kmem_cache *s);
529static inline bool __slub_debug_enabled(void)
530{
531 return static_branch_unlikely(&slub_debug_enabled);
532}
533#else
534static inline void print_tracking(struct kmem_cache *s, void *object)
535{
536}
537static inline bool __slub_debug_enabled(void)
538{
539 return false;
540}
541#endif
542
543/*
544 * Returns true if any of the specified slab_debug flags is enabled for the
545 * cache. Use only for flags parsed by setup_slub_debug() as it also enables
546 * the static key.
547 */
548static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
549{
550 if (IS_ENABLED(CONFIG_SLUB_DEBUG))
551 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
552 if (__slub_debug_enabled())
553 return s->flags & flags;
554 return false;
555}
556
557#if IS_ENABLED(CONFIG_SLUB_DEBUG) && IS_ENABLED(CONFIG_KUNIT)
558bool slab_in_kunit_test(void);
559#else
560static inline bool slab_in_kunit_test(void) { return false; }
561#endif
562
563/*
564 * slub is about to manipulate internal object metadata. This memory lies
565 * outside the range of the allocated object, so accessing it would normally
566 * be reported by kasan as a bounds error. metadata_access_enable() is used
567 * to tell kasan that these accesses are OK.
568 */
569static inline void metadata_access_enable(void)
570{
571 kasan_disable_current();
572 kmsan_disable_current();
573}
574
575static inline void metadata_access_disable(void)
576{
577 kmsan_enable_current();
578 kasan_enable_current();
579}
580
581/*
582 * Return true if KMALLOC_NORMAL caches may need obj_exts arrays.
583 *
584 * Memory allocation profiling requires obj_exts for all caches.
585 * Memcg usually doesn't need them for normal kmalloc caches, but kmalloc types
586 * with a priority higher than KMALLOC_CGROUP can be aliased with KMALLOC_NORMAL.
587 */
588static inline bool need_kmalloc_no_objext(void)
589{
590 if (!mem_alloc_profiling_permanently_disabled())
591 return true;
592
593 if (!mem_cgroup_kmem_disabled() &&
594 (KMALLOC_NORMAL == KMALLOC_RECLAIM))
595 return true;
596
597 return false;
598}
599
600/*
601 * Extended information for slab objects stored as a pointer to an array in
602 * slab->obj_exts (aliasing page->memcg_data) if MEMCG_DATA_OBJEXTS is set.
603 */
604struct slabobj_ext {
605 /*
606 * All elements of the union should be pointer-sized to avoid memory
607 * waste
608 */
609 union {
610#ifdef CONFIG_MEMCG
611 struct obj_cgroup *_objcg;
612#endif
613#ifdef CONFIG_MEM_ALLOC_PROFILING
614 union codetag_ref _ctref;
615#endif
616 };
617} __aligned(8);
618
619#ifdef CONFIG_MEM_ALLOC_PROFILING
620DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
621 slab_obj_ext_has_codetag_key);
622
623static inline bool slab_obj_ext_has_codetag(void)
624{
625 return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
626 &slab_obj_ext_has_codetag_key);
627}
628#else
629static inline bool slab_obj_ext_has_codetag(void)
630{
631 return false;
632}
633#endif
634
635#ifdef CONFIG_MEMCG
636static inline bool cache_needs_objcg(struct kmem_cache *cache)
637{
638 return (cache->flags & SLAB_MAY_ACCOUNT);
639}
640
641static inline bool slab_needs_objcg(struct slab *slab)
642{
643#ifdef CONFIG_64BIT
644 return slab->obj_exts_needs_objcg;
645#else
646 return cache_needs_objcg(slab->slab_cache);
647#endif
648}
649#else
650static inline bool cache_needs_objcg(struct kmem_cache *cache)
651{
652 return false;
653}
654
655static inline bool slab_needs_objcg(struct slab *slab)
656{
657 return false;
658}
659#endif
660
661static inline size_t cache_obj_ext_size(struct kmem_cache *s)
662{
663 size_t sz = 0;
664
665 if (cache_needs_objcg(s))
666 sz += 1;
667
668 if (slab_obj_ext_has_codetag())
669 sz += 1;
670
671 return sizeof(struct slabobj_ext) * sz;
672}
673
674static inline size_t slab_obj_ext_size(struct slab *slab)
675{
676 size_t sz = 0;
677
678 if (slab_needs_objcg(slab))
679 sz += 1;
680
681 if (slab_obj_ext_has_codetag())
682 sz += 1;
683
684 return sizeof(struct slabobj_ext) * sz;
685}
686
687#ifdef CONFIG_SLAB_OBJ_EXT
688
689/*
690 * slab_obj_exts - get the pointer to the slab object extension vector
691 * associated with a slab.
692 * @slab: a pointer to the slab struct
693 *
694 * Returns the address of the object extension vector associated with the slab,
695 * or zero if no such vector has been associated yet.
696 * Do not dereference the return value directly; use get/put_slab_obj_exts()
697 * pair and slab_obj_ext() to access individual elements.
698 *
699 * Example usage:
700 *
701 * obj_exts = slab_obj_exts(slab);
702 * if (obj_exts) {
703 * get_slab_obj_exts(obj_exts);
704 * obj_ext = slab_obj_ext(s, slab, obj_exts, obj);
705 * // do something with obj_ext
706 * put_slab_obj_exts(obj_exts);
707 * }
708 *
709 * Note that the get/put semantics does not involve reference counting.
710 * Instead, it updates kasan/kmsan depth so that accesses to slabobj_ext
711 * won't be reported as access violations.
712 */
713static inline unsigned long slab_obj_exts(struct slab *slab)
714{
715 unsigned long obj_exts = READ_ONCE(slab->obj_exts);
716
717#ifdef CONFIG_MEMCG
718 /*
719 * obj_exts should be either NULL, a valid pointer with
720 * MEMCG_DATA_OBJEXTS bit set or be equal to OBJEXTS_ALLOC_FAIL.
721 */
722 VM_BUG_ON_PAGE(obj_exts && !(obj_exts & MEMCG_DATA_OBJEXTS) &&
723 obj_exts != OBJEXTS_ALLOC_FAIL, slab_page(slab));
724 VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab));
725#endif
726
727 return obj_exts & ~OBJEXTS_FLAGS_MASK;
728}
729
730static inline void get_slab_obj_exts(unsigned long obj_exts)
731{
732 VM_WARN_ON_ONCE(!obj_exts);
733 metadata_access_enable();
734}
735
736static inline void put_slab_obj_exts(unsigned long obj_exts)
737{
738 metadata_access_disable();
739}
740
741#ifdef CONFIG_64BIT
742static inline bool obj_exts_in_object(struct slab *slab)
743{
744 /*
745 * Note we cannot rely on the SLAB_OBJ_EXT_IN_OBJ flag here and need to
746 * check the per-slab bit. A cache can have SLAB_OBJ_EXT_IN_OBJ set, but
747 * allocations within_slab_leftover are preferred. And those may be
748 * possible or not depending on the particular slab's size.
749 */
750 return slab->obj_exts_in_object;
751}
752#else
753static inline bool obj_exts_in_object(struct slab *slab)
754{
755 return false;
756}
757#endif
758
759/*
760 * slab_obj_ext - get the pointer to the slab object extension metadata
761 * associated with an object in a slab.
762 * @s: cache that the slab belongs to
763 * @slab: a pointer to the slab struct
764 * @obj_exts: a pointer to the object extension vector
765 * @obj: a pointer to the object
766 *
767 * Returns a pointer to the object extension associated with the object.
768 * Must be called within a section covered by get/put_slab_obj_exts().
769 */
770static inline struct slabobj_ext *
771slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
772 const void *obj)
773{
774 struct slabobj_ext *obj_ext;
775 unsigned int index;
776 unsigned int stride;
777
778 VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab));
779
780 /*
781 * KFENCE objects have NULL obj_exts and thus can't reach this
782 * and we don't need obj_to_index()
783 */
784 index = __obj_to_index(s, slab_address(slab), obj);
785
786 if (!obj_exts_in_object(slab))
787 stride = slab_obj_ext_size(slab);
788 else
789 stride = s->size;
790
791 obj_ext = (struct slabobj_ext *)(obj_exts + index * stride);
792
793 return kasan_reset_tag(obj_ext);
794}
795
796#ifdef CONFIG_MEMCG
797static inline struct obj_cgroup *
798slab_obj_ext_objcg(struct slab *slab, struct slabobj_ext *obj_ext)
799{
800 VM_WARN_ON_ONCE(!slab_needs_objcg(slab));
801
802 /* if objcg exists, it comes first, so we don't need to do anything */
803 return obj_ext->_objcg;
804}
805
806static inline void
807slab_obj_ext_set_objcg(struct slab *slab, struct slabobj_ext *obj_ext,
808 struct obj_cgroup *objcg)
809{
810 VM_WARN_ON_ONCE(!slab_needs_objcg(slab));
811
812 /* if objcg exists, it comes first, so we don't need to do anything */
813 obj_ext->_objcg = objcg;
814}
815#endif
816
817#ifdef CONFIG_MEM_ALLOC_PROFILING
818static inline union codetag_ref *
819slab_obj_ext_codetag_ref(struct slab *slab, struct slabobj_ext *obj_ext)
820{
821 VM_WARN_ON_ONCE(!slab_obj_ext_has_codetag());
822
823 if (slab_needs_objcg(slab))
824 obj_ext += 1;
825
826 return &obj_ext->_ctref;
827}
828#endif
829
830int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
831 gfp_t gfp, unsigned int alloc_flags);
832
833#else /* CONFIG_SLAB_OBJ_EXT */
834
835static inline unsigned long slab_obj_exts(struct slab *slab)
836{
837 return 0;
838}
839
840static inline struct slabobj_ext *
841slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts,
842 const void *obj)
843{
844 return NULL;
845}
846
847static inline bool obj_exts_in_object(struct slab *slab)
848{
849 return false;
850}
851
852#endif /* CONFIG_SLAB_OBJ_EXT */
853
854static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
855{
856 return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
857 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
858}
859
860#ifdef CONFIG_MEMCG
861bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
862 gfp_t flags, unsigned int slab_alloc_flags,
863 size_t size, void **p);
864void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
865 void **p, int objects, unsigned long obj_exts);
866#endif
867
868void kvfree_rcu_cb(struct rcu_head *head);
869
870static inline unsigned int large_kmalloc_order(const struct page *page)
871{
872 return page[1].flags.f & 0xff;
873}
874
875static inline size_t large_kmalloc_size(const struct page *page)
876{
877 return PAGE_SIZE << large_kmalloc_order(page);
878}
879
880#ifdef CONFIG_SLUB_DEBUG
881void dump_unreclaimable_slab(void);
882#else
883static inline void dump_unreclaimable_slab(void)
884{
885}
886#endif
887
888void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
889
890#ifdef CONFIG_SLAB_FREELIST_RANDOM
891int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
892 gfp_t gfp);
893void cache_random_seq_destroy(struct kmem_cache *cachep);
894#else
895static inline int cache_random_seq_create(struct kmem_cache *cachep,
896 unsigned int count, gfp_t gfp)
897{
898 return 0;
899}
900static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
901#endif /* CONFIG_SLAB_FREELIST_RANDOM */
902
903static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
904{
905 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
906 &init_on_alloc)) {
907 if (c->ctor)
908 return false;
909 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
910 return flags & __GFP_ZERO;
911 return true;
912 }
913 return flags & __GFP_ZERO;
914}
915
916static inline bool slab_want_init_on_free(struct kmem_cache *c)
917{
918 if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
919 &init_on_free))
920 return !(c->ctor ||
921 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
922 return false;
923}
924
925#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
926void debugfs_slab_release(struct kmem_cache *);
927#else
928static inline void debugfs_slab_release(struct kmem_cache *s) { }
929#endif
930
931#ifdef CONFIG_PRINTK
932#define KS_ADDRS_COUNT 16
933struct kmem_obj_info {
934 void *kp_ptr;
935 struct slab *kp_slab;
936 void *kp_objp;
937 unsigned long kp_data_offset;
938 struct kmem_cache *kp_slab_cache;
939 void *kp_ret;
940 void *kp_stack[KS_ADDRS_COUNT];
941 void *kp_free_stack[KS_ADDRS_COUNT];
942};
943void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);
944#endif
945
946void __check_heap_object(const void *ptr, unsigned long n,
947 const struct slab *slab, bool to_user);
948
949void deferred_work_barrier(void);
950void defer_kfree_rcu(struct kvfree_rcu_head *head);
951
952static inline bool slub_debug_orig_size(struct kmem_cache *s)
953{
954 return (kmem_cache_debug_flags(s, SLAB_STORE_USER) &&
955 (s->flags & SLAB_KMALLOC));
956}
957
958#ifdef CONFIG_SLUB_DEBUG
959void skip_orig_size_check(struct kmem_cache *s, const void *object);
960#endif
961
962#endif /* MM_SLAB_H */