projects/linux·commit 05cef13f

Merge tag 'slab-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

05cef13f · Linus Torvalds <[email protected]> · 2026-04-15 10:15:39 -07:00

Pull slab updates from Vlastimil Babka:

 - Sheaves performance improvements for systems with memoryless NUMA
   nodes, developed in response to regression reports.

   These mainly ensure that percpu sheaves exist and are used on cpus
   that belong to these memoryless nodes (Vlastimil Babka, Hao Li).

 - Cleanup API usage and constify sysfs attributes (Thomas Weißschuh)

 - Disable kfree_rcu() batching on builds intended for fuzzing/debugging
   that enable CONFIG_RCU_STRICT_GRACE_PERIOD (Jann Horn)

 - Add a kunit test for kmalloc_nolock()/kfree_nolock() (Harry Yoo)

* tag 'slab-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
  slub: clarify kmem_cache_refill_sheaf() comments
  lib/tests/slub_kunit: add a test case for {kmalloc,kfree}_nolock
  MAINTAINERS: add lib/tests/slub_kunit.c to SLAB ALLOCATOR section
  slub: use N_NORMAL_MEMORY in can_free_to_pcs to handle remote frees
  slab,rcu: disable KVFREE_RCU_BATCHED for strict grace period
  slab: free remote objects to sheaves on memoryless nodes
  slab: create barns for online memoryless nodes
  slab: decouple pointer to barn from kmem_cache_node
  slab: remove alloc_full_sheaf()
  mm/slab: constify sysfs attributes
  mm/slab: create sysfs attribute through default_groups
Commit
05cef13fa80de8cec481ae5a015e58bc6340ca2d
Parents
a8e7ef3c · 44e0ebe4

Merge commit. Changes below are compared with the first parent: a8e7ef3c.

Changes

5 files changed · +322 −130

FileStatusAddDelContent
MAINTAINERSmodified+1−0View file
lib/tests/slub_kunit.cmodified+92−0View file
mm/Kconfigmodified+1−0View file
mm/slab.hmodified+6−1View file
mm/slub.cmodified+222−129View file

Only the first 600 diff lines are shown. Download the complete patch.

MAINTAINERS

View file
diff --git a/MAINTAINERS b/MAINTAINERS
index c8f6bc6..e6a7360 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -24495,6 +24495,7 @@ F: Documentation/admin-guide/mm/slab.rst
F: Documentation/mm/slab.rst
F: include/linux/mempool.h
F: include/linux/slab.h
+F: lib/tests/slub_kunit.c
F: mm/failslab.c
F: mm/mempool.c
F: mm/slab.h

lib/tests/slub_kunit.c

View file
diff --git a/lib/tests/slub_kunit.c b/lib/tests/slub_kunit.c
index 848b682..fa6d31d 100644
--- a/lib/tests/slub_kunit.c
+++ b/lib/tests/slub_kunit.c
@@ -7,6 +7,7 @@
#include <linux/kernel.h>
#include <linux/rcupdate.h>
#include <linux/delay.h>
+#include <linux/perf_event.h>
#include "../mm/slab.h"
static struct kunit_resource resource;
@@ -291,6 +292,94 @@ static void test_krealloc_redzone_zeroing(struct kunit *test)
kmem_cache_destroy(s);
}
+#ifdef CONFIG_PERF_EVENTS
+#define NR_ITERATIONS 1000
+#define NR_OBJECTS 1000
+static void *objects[NR_OBJECTS];
+
+struct test_nolock_context {
+ struct kunit *test;
+ int callback_count;
+ int alloc_ok;
+ int alloc_fail;
+ struct perf_event *event;
+};
+
+static struct perf_event_attr hw_attr = {
+ .type = PERF_TYPE_HARDWARE,
+ .config = PERF_COUNT_HW_CPU_CYCLES,
+ .size = sizeof(struct perf_event_attr),
+ .pinned = 1,
+ .disabled = 1,
+ .freq = 1,
+ .sample_freq = 100000,
+};
+
+static void overflow_handler_test_kmalloc_kfree_nolock(struct perf_event *event,
+ struct perf_sample_data *data,
+ struct pt_regs *regs)
+{
+ void *objp;
+ gfp_t gfp;
+ struct test_nolock_context *ctx = event->overflow_handler_context;
+
+ /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */
+ gfp = (ctx->callback_count % 2) ? 0 : __GFP_ACCOUNT;
+ objp = kmalloc_nolock(64, gfp, NUMA_NO_NODE);
+
+ if (objp)
+ ctx->alloc_ok++;
+ else
+ ctx->alloc_fail++;
+
+ kfree_nolock(objp);
+ ctx->callback_count++;
+}
+
+static void test_kmalloc_kfree_nolock(struct kunit *test)
+{
+ int i, j;
+ struct test_nolock_context ctx = { .test = test };
+ struct perf_event *event;
+ bool alloc_fail = false;
+
+ event = perf_event_create_kernel_counter(&hw_attr, -1, current,
+ overflow_handler_test_kmalloc_kfree_nolock,
+ &ctx);
+ if (IS_ERR(event))
+ kunit_skip(test, "Failed to create perf event");
+ ctx.event = event;
+ perf_event_enable(ctx.event);
+ for (i = 0; i < NR_ITERATIONS; i++) {
+ for (j = 0; j < NR_OBJECTS; j++) {
+ gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT;
+
+ objects[j] = kmalloc(64, gfp);
+ if (!objects[j]) {
+ j--;
+ while (j >= 0)
+ kfree(objects[j--]);
+ alloc_fail = true;
+ goto cleanup;
+ }
+ }
+ for (j = 0; j < NR_OBJECTS; j++)
+ kfree(objects[j]);
+ }
+
+cleanup:
+ perf_event_disable(ctx.event);
+ perf_event_release_kernel(ctx.event);
+
+ kunit_info(test, "callback_count: %d, alloc_ok: %d, alloc_fail: %d\n",
+ ctx.callback_count, ctx.alloc_ok, ctx.alloc_fail);
+
+ if (alloc_fail)
+ kunit_skip(test, "Allocation failed");
+ KUNIT_EXPECT_EQ(test, 0, slab_errors);
+}
+#endif
+
static int test_init(struct kunit *test)
{
slab_errors = 0;
@@ -315,6 +404,9 @@ static struct kunit_case test_cases[] = {
KUNIT_CASE(test_kfree_rcu_wq_destroy),
KUNIT_CASE(test_leak_destroy),
KUNIT_CASE(test_krealloc_redzone_zeroing),
+#ifdef CONFIG_PERF_EVENTS
+ KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock),
+#endif
{}
};

mm/Kconfig

View file
diff --git a/mm/Kconfig b/mm/Kconfig
index ebd8ea3..67a72fe 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -172,6 +172,7 @@ config SLUB
config KVFREE_RCU_BATCHED
def_bool y
depends on !SLUB_TINY && !TINY_RCU
+ depends on !RCU_STRICT_GRACE_PERIOD
config SLUB_TINY
bool "Configure for minimal memory footprint"

mm/slab.h

View file
diff --git a/mm/slab.h b/mm/slab.h
index e9ab292..c735e6b 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -191,6 +191,11 @@ struct kmem_cache_order_objects {
unsigned int x;
};
+struct kmem_cache_per_node_ptrs {
+ struct node_barn *barn;
+ struct kmem_cache_node *node;
+};
+
/*
* Slab cache management.
*/
@@ -247,7 +252,7 @@ struct kmem_cache {
struct kmem_cache_stats __percpu *cpu_stats;
#endif
- struct kmem_cache_node *node[MAX_NUMNODES];
+ struct kmem_cache_per_node_ptrs per_node[MAX_NUMNODES];
};
/*

mm/slub.c

View file
diff --git a/mm/slub.c b/mm/slub.c
index 2b2d33c..92362ee 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -59,7 +59,7 @@
* 0. cpu_hotplug_lock
* 1. slab_mutex (Global Mutex)
* 2a. kmem_cache->cpu_sheaves->lock (Local trylock)
- * 2b. node->barn->lock (Spinlock)
+ * 2b. barn->lock (Spinlock)
* 2c. node->list_lock (Spinlock)
* 3. slab_lock(slab) (Only on some arches)
* 4. object_map_lock (Only for debugging)
@@ -136,7 +136,7 @@
* or spare sheaf can handle the allocation or free, there is no other
* overhead.
*
- * node->barn->lock (spinlock)
+ * barn->lock (spinlock)
*
* This lock protects the operations on per-NUMA-node barn. It can quickly
* serve an empty or full sheaf if available, and avoid more expensive refill
@@ -436,26 +436,24 @@ struct kmem_cache_node {
atomic_long_t total_objects;
struct list_head full;
#endif
- struct node_barn *barn;
};
static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
{
- return s->node[node];
+ return s->per_node[node].node;
+}
+
+static inline struct node_barn *get_barn_node(struct kmem_cache *s, int node)
+{
+ return s->per_node[node].barn;
}
/*
- * Get the barn of the current cpu's closest memory node. It may not exist on
- * systems with memoryless nodes but without CONFIG_HAVE_MEMORYLESS_NODES
+ * Get the barn of the current cpu's NUMA node. It may be a memoryless node.
*/
static inline struct node_barn *get_barn(struct kmem_cache *s)
{
- struct kmem_cache_node *n = get_node(s, numa_mem_id());
-
- if (!n)
- return NULL;
-
- return n->barn;
+ return get_barn_node(s, numa_node_id());
}
/*
@@ -475,6 +473,12 @@ static inline struct node_barn *get_barn(struct kmem_cache *s)
static nodemask_t slab_nodes;
/*
+ * Similar to slab_nodes but for where we have node_barn allocated.
+ * Corresponds to N_ONLINE nodes.
+ */
+static nodemask_t slab_barn_nodes;
+
+/*
* Workqueue used for flushing cpu and kfree_rcu sheaves.
*/
static struct workqueue_struct *flushwq;
@@ -2822,24 +2826,6 @@ static int refill_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf,
return 0;
}
-static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf);
-
-static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp)
-{
- struct slab_sheaf *sheaf = alloc_empty_sheaf(s, gfp);
-
- if (!sheaf)
- return NULL;
-
- if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
- sheaf_flush_unused(s, sheaf);
- free_empty_sheaf(s, sheaf);
- return NULL;
- }
-
- return sheaf;
-}
-
/*
* Maximum number of objects freed during a single flush of main pcs sheaf.
* Translates directly to an on-stack array size.
@@ -4082,6 +4068,51 @@ void flush_all_rcu_sheaves(void)
rcu_barrier();
}
+static int slub_cpu_setup(unsigned int cpu)
+{
+ int nid = cpu_to_node(cpu);
+ struct kmem_cache *s;
+ int ret = 0;
+
+ /*
+ * we never clear a nid so it's safe to do a quick check before taking
+ * the mutex, and then recheck to handle parallel cpu hotplug safely
+ */
+ if (node_isset(nid, slab_barn_nodes))
+ return 0;
+
+ mutex_lock(&slab_mutex);
+
+ if (node_isset(nid, slab_barn_nodes))
+ goto out;
+
+ list_for_each_entry(s, &slab_caches, list) {
+ struct node_barn *barn;
+
+ /*
+ * barn might already exist if a previous callback failed midway
+ */
+ if (!cache_has_sheaves(s) || get_barn_node(s, nid))
+ continue;
+
+ barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, nid);
+
+ if (!barn) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ barn_init(barn);
+ s->per_node[nid].barn = barn;
+ }
+ node_set(nid, slab_barn_nodes);
+
+out:
+ mutex_unlock(&slab_mutex);
+
+ return ret;
+}
+
/*
* Use the cpu notifier to insure that the cpu slabs are flushed when
* necessary.
@@ -4611,34 +4642,35 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
if (!allow_spin)
return NULL;
- if (empty) {
- if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
- full = empty;
- } else {
- /*
- * we must be very low on memory so don't bother
- * with the barn
- */
- sheaf_flush_unused(s, empty);
- free_empty_sheaf(s, empty);
- }
- } else {
- full = alloc_full_sheaf(s, gfp);
+ if (!empty) {
+ empty = alloc_empty_sheaf(s, gfp);
+ if (!empty)
+ return NULL;
}
- if (!full)
+ if (refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
+ /*
+ * we must be very low on memory so don't bother
+ * with the barn
+ */
+ sheaf_flush_unused(s, empty);
+ free_empty_sheaf(s, empty);
+
return NULL;
+ }
+
+ full = empty;
+ empty = NULL;
if (!local_trylock(&s->cpu_sheaves->lock))
goto barn_put;
pcs = this_cpu_ptr(s->cpu_sheaves);
/*
- * If we are returning empty sheaf, we either got it from the
- * barn or had to allocate one. If we are returning a full
- * sheaf, it's due to racing or being migrated to a different
- * cpu. Breaching the barn's sheaf limits should be thus rare
- * enough so just ignore them to simplify the recovery.
+ * If we put any empty or full sheaf to the barn below, it's due to
+ * racing or being migrated to a different cpu. Breaching the barn's
+ * sheaf limits should be thus rare enough so just ignore them to
+ * simplify the recovery.
*/
if (pcs->main->size == 0) {
@@ -5088,12 +5120,15 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
}
/*
- * refill a sheaf previously returned by kmem_cache_prefill_sheaf to at least
- * the given size
+ * Refill a sheaf previously returned by kmem_cache_prefill_sheaf to at least
+ * the given size.
*
- * the sheaf might be replaced by a new one when requesting more than
- * s->sheaf_capacity objects if such replacement is necessary, but the refill
- * fails (returning -ENOMEM), the existing sheaf is left intact
+ * Return: 0 on success. The sheaf will contain at least @size objects.
+ * The sheaf might have been replaced with a new one if more than
+ * sheaf->capacity objects are requested.
+ *
+ * Return: -ENOMEM on failure. Some objects might have been added to the sheaf
+ * but the sheaf will not be replaced.
*
* In practice we always refill to full sheaf's capacity.
*/
@@ -5788,7 +5823,6 @@ bool free_to_pcs(struct kmem_cache *s, void *object, bool allow_spin)
static void rcu_free_sheaf(struct rcu_head *head)
{
- struct kmem_cache_node *n;
struct slab_sheaf *sheaf;
struct node_barn *barn = NULL;
struct kmem_cache *s;
@@ -5811,12 +5845,10 @@ static void rcu_free_sheaf(struct rcu_head *head)
if (__rcu_free_sheaf_prepare(s, sheaf))
goto flush;
- n = get_node(s, sheaf->node);
- if (!n)
+ barn = get_barn_node(s, sheaf->node);
+ if (!barn)
goto flush;
- barn = n->barn;
-
/* due to slab_free_hook() */
if (unlikely(sheaf->size == 0))
goto empty;
@@ -5938,7 +5970,7 @@ do_free:
rcu_sheaf = NULL;
} else {
pcs->rcu_free = NULL;
- rcu_sheaf->node = numa_mem_id();
+ rcu_sheaf->node = numa_node_id();
}
/*
@@ -5960,6 +5992,57 @@ fail:
return false;
}
+static __always_inline bool can_free_to_pcs(struct slab *slab)
+{
+ int slab_node;
+ int numa_node;
+
+ if (!IS_ENABLED(CONFIG_NUMA))
+ goto check_pfmemalloc;
+
+ slab_node = slab_nid(slab);
+
+#ifdef CONFIG_HAVE_MEMORYLESS_NODES
+ /*
+ * numa_mem_id() points to the closest node with memory so only allow
+ * objects from that node to the percpu sheaves
+ */
+ numa_node = numa_mem_id();
+
+ if (likely(slab_node == numa_node))
+ goto check_pfmemalloc;
+#else
+
+ /*
+ * numa_mem_id() is only a wrapper to numa_node_id() which is where this
+ * cpu belongs to, but it might be a memoryless node anyway. We don't
+ * know what the closest node is.
+ */
+ numa_node = numa_node_id();
+
+ /* freed object is from this cpu's node, proceed */
+ if (likely(slab_node == numa_node))
+ goto check_pfmemalloc;
+
+ /*
+ * Freed object isn't from this cpu's node, but that node is memoryless
+ * or only has ZONE_MOVABLE memory, which slab cannot allocate from.
+ * Proceed as it's better to cache remote objects than falling back to
+ * the slowpath for everything. The allocation side can never obtain
+ * a local object anyway, if none exist. We don't have numa_mem_id() to
+ * point to the closest node as we would on a proper memoryless node
+ * setup.
+ */
+ if (unlikely(!node_state(numa_node, N_NORMAL_MEMORY)))
+ goto check_pfmemalloc;
+#endif
+
+ return false;
+
+check_pfmemalloc:
+ return likely(!slab_test_pfmemalloc(slab));
+}
+
/*
* Bulk free objects to the percpu sheaves.
* Unlike free_to_pcs() this includes the calls to all necessary hooks
@@ -5974,7 +6057,6 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
struct node_barn *barn;
void *remote_objects[PCS_BATCH_MAX];
unsigned int remote_nr = 0;
- int node = numa_mem_id();
next_remote_batch:
while (i < size) {
@@ -5988,8 +6070,7 @@ next_remote_batch:
continue;
}
- if (unlikely((IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node)
- || slab_test_pfmemalloc(slab))) {
+ if (unlikely(!can_free_to_pcs(slab))) {
remote_objects[remote_nr] = p[i];
p[i] = p[--size];
if (++remote_nr >= PCS_BATCH_MAX)
@@ -6165,11 +6246,8 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false)))
return;
- if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())
- && likely(!slab_test_pfmemalloc(slab))) {
- if (likely(free_to_pcs(s, object, true)))
- return;
- }
+ if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true)))
+ return;
__slab_free(s, slab, object, object, 1, addr);
stat(s, FREE_SLOWPATH);
@@ -6540,10 +6618,8 @@ void kfree_nolock(const void *object)
*/
kasan_slab_free(s, x, false, false, /* skip quarantine */true);
- if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) {
- if (likely(free_to_pcs(s, x, false)))
- return;
- }
+ if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false)))
+ return;
/*
* __slab_free() can locklessly cmpxchg16 into a slab, but then it might
@@ -7427,7 +7503,7 @@ static inline int calculate_order(unsigned int size)
}
static void
-init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn)
+init_kmem_cache_node(struct kmem_cache_node *n)
{
n->nr_partial = 0;
spin_lock_init(&n->list_lock);
@@ -7437,9 +7513,6 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn)
atomic_long_set(&n->total_objects, 0);
INIT_LIST_HEAD(&n->full);
#endif
- n->barn = barn;
- if (barn)
- barn_init(barn);
}
#ifdef CONFIG_SLUB_STATS
@@ -7534,8 +7607,8 @@ static void early_kmem_cache_node_alloc(int node)
n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
slab->freelist = get_freepointer(kmem_cache_node, n);
slab->inuse = 1;
- kmem_cache_node->node[node] = n;
- init_kmem_cache_node(n, NULL);
+ kmem_cache_node->per_node[node].node = n;
+ init_kmem_cache_node(n);
inc_slabs_node(kmem_cache_node, node, slab->objects);
/*
@@ -7550,15 +7623,20 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
int node;
struct kmem_cache_node *n;
- for_each_kmem_cache_node(s, node, n) {
- if (n->barn) {
- WARN_ON(n->barn->nr_full);
- WARN_ON(n->barn->nr_empty);
- kfree(n->barn);
- n->barn = NULL;
- }
+ for_each_node(node) {
+ struct node_barn *barn = get_barn_node(s, node);
+
+ if (!barn)
+ continue;
+
+ WARN_ON(barn->nr_full);
+ WARN_ON(barn->nr_empty);
+ kfree(barn);
+ s->per_node[node].barn = NULL;
+ }
- s->node[node] = NULL;
+ for_each_kmem_cache_node(s, node, n) {
+ s->per_node[node].node = NULL;
kmem_cache_free(kmem_cache_node, n);
}
}
@@ -7579,31 +7657,36 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)
for_each_node_mask(node, slab_nodes) {
struct kmem_cache_node *n;
- struct node_barn *barn = NULL;
if (slab_state == DOWN) {
early_kmem_cache_node_alloc(node);
continue;
}
- if (cache_has_sheaves(s)) {
- barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
-
- if (!barn)
- return 0;
- }

This diff is incomplete. Download the complete patch.