From efa6a5b7bbb1bbdf304afc4a21da8ebacc2f758a Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 13 Jul 2026 09:52:30 +0200 Subject: [PATCH] mm/slab: simplify freeing remote objects in free_to_pcs_bulk() We have been moving remote objects to an on-stack array and flushing it when full. Instead, we can swap them towards the beginning of the supplied array and bulk-free it just once. Also add a comment to explain the rationale of freeing remote objects last, because now it would appear to be simpler to free them first. Reviewed-by: Pedro Falcato Reviewed-by: Shengming Hu Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260713-bulk_free_remote-v2-1-24ee24771c2f@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 22045dc919ef..b650c1729437 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6255,9 +6255,17 @@ no_empty: static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) { bool init = slab_want_init_on_free(s); - void *remote_objects[PCS_BATCH_MAX]; + void **remote_objects = p; unsigned int remote_nr = 0; + /* + * Process the free hooks and separate out remote objects by + * partitioning the 'p' array in place: + * + * [0, remote_nr) - processed remote objects + * [remote_nr, i) - processed local objects + * [i, size) - unprocessed objects + */ for (unsigned int i = 0; i < size;) { struct slab *slab = virt_to_slab(p[i]); @@ -6270,19 +6278,17 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) } if (unlikely(!can_free_to_pcs(slab))) { - remote_objects[remote_nr] = p[i]; - p[i] = p[--size]; - if (++remote_nr >= PCS_BATCH_MAX) { - __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); - stat_add(s, FREE_SLOWPATH, remote_nr); - remote_nr = 0; - } - continue; + if (i != remote_nr) + swap(remote_objects[remote_nr], p[i]); + remote_nr++; } i++; } + p += remote_nr; + size -= remote_nr; + while (size) { unsigned int batch_freed = __free_to_pcs_batch(s, size, p); @@ -6296,8 +6302,12 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) size -= batch_freed; } + /* + * Processing remote objects last decreases the chances of cpu migration + * while freeing to sheaves and compromising object locality + */ if (remote_nr) { - __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); + __kmem_cache_free_bulk(s, remote_nr, remote_objects); stat_add(s, FREE_SLOWPATH, remote_nr); } }