Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'slab/for-6.19/sheaves_cleanups' into slab/for-next 

Merge series "slab: preparatory cleanups before adding sheaves to all
caches" [1]

Cleanups that were written as part of the full sheaves conversion, which
is not fully ready yet, but they are useful on their own.

Link: https://lore.kernel.org/all/20251105-sheaves-cleanups-v1-0-b8218e1ac7ef@suse.cz/ [1]
This commit is contained in:
Vlastimil Babka 2025-11-25 14:27:34 +01:00
parent 8ad018dbd3 6adf4b11fa
commit 3065c20d5d
3 changed files with 173 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/gfp_types.h
View File

@ -55,9 +55,7 @@ enum {
#ifdef CONFIG_LOCKDEP #ifdef CONFIG_LOCKDEP
___GFP_NOLOCKDEP_BIT, ___GFP_NOLOCKDEP_BIT,
#endif #endif
#ifdef CONFIG_SLAB_OBJ_EXT
___GFP_NO_OBJ_EXT_BIT, ___GFP_NO_OBJ_EXT_BIT,
#endif
___GFP_LAST_BIT ___GFP_LAST_BIT
}; };
@ -98,11 +96,7 @@ enum {
#else #else
#define ___GFP_NOLOCKDEP 0 #define ___GFP_NOLOCKDEP 0
#endif #endif
#ifdef CONFIG_SLAB_OBJ_EXT
#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) #define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT)
#else
#define ___GFP_NO_OBJ_EXT 0
#endif
/* /*
* Physical address zone modifiers (see linux/mmzone.h - low four bits) * Physical address zone modifiers (see linux/mmzone.h - low four bits)

2
mm/slab.h
View File

@ -236,10 +236,8 @@ struct kmem_cache_order_objects {
* Slab cache management. * Slab cache management.
*/ */
struct kmem_cache { struct kmem_cache {
#ifndef CONFIG_SLUB_TINY
struct kmem_cache_cpu __percpu *cpu_slab; struct kmem_cache_cpu __percpu *cpu_slab;
struct lock_class_key lock_key; struct lock_class_key lock_key;
#endif
struct slub_percpu_sheaves __percpu *cpu_sheaves; struct slub_percpu_sheaves __percpu *cpu_sheaves;
/* Used for retrieving partial slabs, etc. */ /* Used for retrieving partial slabs, etc. */
slab_flags_t flags; slab_flags_t flags;

331
mm/slub.c
View File

@ -410,7 +410,6 @@ enum stat_item {
NR_SLUB_STAT_ITEMS NR_SLUB_STAT_ITEMS
}; };
#ifndef CONFIG_SLUB_TINY
/* /*
* When changing the layout, make sure freelist and tid are still compatible * When changing the layout, make sure freelist and tid are still compatible
* with this_cpu_cmpxchg_double() alignment requirements. * with this_cpu_cmpxchg_double() alignment requirements.
@ -432,7 +431,6 @@ struct kmem_cache_cpu {
unsigned int stat[NR_SLUB_STAT_ITEMS]; unsigned int stat[NR_SLUB_STAT_ITEMS];
#endif #endif
}; };
#endif /* CONFIG_SLUB_TINY */
static inline void stat(const struct kmem_cache *s, enum stat_item si) static inline void stat(const struct kmem_cache *s, enum stat_item si)
{ {
@ -469,7 +467,10 @@ struct slab_sheaf {
struct rcu_head rcu_head; struct rcu_head rcu_head;
struct list_head barn_list; struct list_head barn_list;
/* only used for prefilled sheafs */ /* only used for prefilled sheafs */
unsigned int capacity; struct {
unsigned int capacity;
bool pfmemalloc;
};
}; };
struct kmem_cache *cache; struct kmem_cache *cache;
unsigned int size; unsigned int size;
@ -594,12 +595,10 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
return freelist_ptr_decode(s, p, ptr_addr); return freelist_ptr_decode(s, p, ptr_addr);
} }
#ifndef CONFIG_SLUB_TINY
static void prefetch_freepointer(const struct kmem_cache *s, void *object) static void prefetch_freepointer(const struct kmem_cache *s, void *object)
{ {
prefetchw(object + s->offset); prefetchw(object + s->offset);
} }
#endif
/* /*
* When running under KMSAN, get_freepointer_safe() may return an uninitialized * When running under KMSAN, get_freepointer_safe() may return an uninitialized
@ -711,10 +710,12 @@ static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s)
return s->cpu_partial_slabs; return s->cpu_partial_slabs;
} }
#else #else
#ifdef SLAB_SUPPORTS_SYSFS
static inline void static inline void
slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
{ {
} }
#endif
static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s)
{ {
@ -2027,15 +2028,21 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node,
int objects) {} int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node, static inline void dec_slabs_node(struct kmem_cache *s, int node,
int objects) {} int objects) {}
#ifndef CONFIG_SLUB_TINY
static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
void **freelist, void *nextfree) void **freelist, void *nextfree)
{ {
return false; return false;
} }
#endif
#endif /* CONFIG_SLUB_DEBUG */ #endif /* CONFIG_SLUB_DEBUG */
/*
* The allocated objcg pointers array is not accounted directly.
* Moreover, it should not come from DMA buffer and is not readily
* reclaimable. So those GFP bits should be masked off.
*/
#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \
__GFP_ACCOUNT | __GFP_NOFAIL)
#ifdef CONFIG_SLAB_OBJ_EXT #ifdef CONFIG_SLAB_OBJ_EXT
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
@ -2086,14 +2093,6 @@ static inline void handle_failed_objexts_alloc(unsigned long obj_exts,
#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ #endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
/*
* The allocated objcg pointers array is not accounted directly.
* Moreover, it should not come from DMA buffer and is not readily
* reclaimable. So those GFP bits should be masked off.
*/
#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \
__GFP_ACCOUNT | __GFP_NOFAIL)
static inline void init_slab_obj_exts(struct slab *slab) static inline void init_slab_obj_exts(struct slab *slab)
{ {
slab->obj_exts = 0; slab->obj_exts = 0;
@ -2601,8 +2600,24 @@ static void *setup_object(struct kmem_cache *s, void *object)
static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
{ {
struct slab_sheaf *sheaf = kzalloc(struct_size(sheaf, objects, struct slab_sheaf *sheaf;
s->sheaf_capacity), gfp); size_t sheaf_size;
if (gfp & __GFP_NO_OBJ_EXT)
return NULL;
gfp &= ~OBJCGS_CLEAR_MASK;
/*
* Prevent recursion to the same cache, or a deep stack of kmallocs of
* varying sizes (sheaf capacity might differ for each kmalloc size
* bucket)
*/
if (s->flags & SLAB_KMALLOC)
gfp |= __GFP_NO_OBJ_EXT;
sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);
sheaf = kzalloc(sheaf_size, gfp);
if (unlikely(!sheaf)) if (unlikely(!sheaf))
return NULL; return NULL;
@ -2655,7 +2670,7 @@ static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp)
if (!sheaf) if (!sheaf)
return NULL; return NULL;
if (refill_sheaf(s, sheaf, gfp)) { if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC)) {
free_empty_sheaf(s, sheaf); free_empty_sheaf(s, sheaf);
return NULL; return NULL;
} }
@ -2733,12 +2748,13 @@ static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf)
sheaf->size = 0; sheaf->size = 0;
} }
static void __rcu_free_sheaf_prepare(struct kmem_cache *s, static bool __rcu_free_sheaf_prepare(struct kmem_cache *s,
struct slab_sheaf *sheaf) struct slab_sheaf *sheaf)
{ {
bool init = slab_want_init_on_free(s); bool init = slab_want_init_on_free(s);
void **p = &sheaf->objects[0]; void **p = &sheaf->objects[0];
unsigned int i = 0; unsigned int i = 0;
bool pfmemalloc = false;
while (i < sheaf->size) { while (i < sheaf->size) {
struct slab *slab = virt_to_slab(p[i]); struct slab *slab = virt_to_slab(p[i]);
@ -2751,8 +2767,13 @@ static void __rcu_free_sheaf_prepare(struct kmem_cache *s,
continue; continue;
} }
if (slab_test_pfmemalloc(slab))
pfmemalloc = true;
i++; i++;
} }
return pfmemalloc;
} }
static void rcu_free_sheaf_nobarn(struct rcu_head *head) static void rcu_free_sheaf_nobarn(struct rcu_head *head)
@ -3015,14 +3036,11 @@ static void barn_init(struct node_barn *barn)
static void barn_shrink(struct kmem_cache *s, struct node_barn *barn) static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)
{ {
struct list_head empty_list; LIST_HEAD(empty_list);
struct list_head full_list; LIST_HEAD(full_list);
struct slab_sheaf *sheaf, *sheaf2; struct slab_sheaf *sheaf, *sheaf2;
unsigned long flags; unsigned long flags;
INIT_LIST_HEAD(&empty_list);
INIT_LIST_HEAD(&full_list);
spin_lock_irqsave(&barn->lock, flags); spin_lock_irqsave(&barn->lock, flags);
list_splice_init(&barn->sheaves_full, &full_list); list_splice_init(&barn->sheaves_full, &full_list);
@ -3618,8 +3636,6 @@ static struct slab *get_partial(struct kmem_cache *s, int node,
return get_any_partial(s, pc); return get_any_partial(s, pc);
} }
#ifndef CONFIG_SLUB_TINY
#ifdef CONFIG_PREEMPTION #ifdef CONFIG_PREEMPTION
/* /*
* Calculate the next globally unique transaction for disambiguation * Calculate the next globally unique transaction for disambiguation
@ -4019,12 +4035,6 @@ static bool has_cpu_slab(int cpu, struct kmem_cache *s)
return c->slab || slub_percpu_partial(c); return c->slab || slub_percpu_partial(c);
} }
#else /* CONFIG_SLUB_TINY */
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { }
static inline bool has_cpu_slab(int cpu, struct kmem_cache *s) { return false; }
static inline void flush_this_cpu_slab(struct kmem_cache *s) { }
#endif /* CONFIG_SLUB_TINY */
static bool has_pcs_used(int cpu, struct kmem_cache *s) static bool has_pcs_used(int cpu, struct kmem_cache *s)
{ {
struct slub_percpu_sheaves *pcs; struct slub_percpu_sheaves *pcs;
@ -4365,7 +4375,6 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
return true; return true;
} }
#ifndef CONFIG_SLUB_TINY
static inline bool static inline bool
__update_cpu_freelist_fast(struct kmem_cache *s, __update_cpu_freelist_fast(struct kmem_cache *s,
void *freelist_old, void *freelist_new, void *freelist_old, void *freelist_new,
@ -4629,7 +4638,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
pc.orig_size = orig_size; pc.orig_size = orig_size;
slab = get_partial(s, node, &pc); slab = get_partial(s, node, &pc);
if (slab) { if (slab) {
if (kmem_cache_debug(s)) { if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
freelist = pc.object; freelist = pc.object;
/* /*
* For debug caches here we had to go through * For debug caches here we had to go through
@ -4667,11 +4676,15 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
stat(s, ALLOC_SLAB); stat(s, ALLOC_SLAB);
if (kmem_cache_debug(s)) { if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags);
if (unlikely(!freelist)) if (unlikely(!freelist)) {
/* This could cause an endless loop. Fail instead. */
if (!allow_spin)
return NULL;
goto new_objects; goto new_objects;
}
if (s->flags & SLAB_STORE_USER) if (s->flags & SLAB_STORE_USER)
set_track(s, freelist, TRACK_ALLOC, addr, set_track(s, freelist, TRACK_ALLOC, addr,
@ -4875,32 +4888,6 @@ static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
return object; return object;
} }
#else /* CONFIG_SLUB_TINY */
static void *__slab_alloc_node(struct kmem_cache *s,
gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
{
struct partial_context pc;
struct slab *slab;
void *object;
pc.flags = gfpflags;
pc.orig_size = orig_size;
slab = get_partial(s, node, &pc);
if (slab)
return pc.object;
slab = new_slab(s, gfpflags, node);
if (unlikely(!slab)) {
slab_out_of_memory(s, gfpflags, node);
return NULL;
}
object = alloc_single_from_new_slab(s, slab, orig_size, gfpflags);
return object;
}
#endif /* CONFIG_SLUB_TINY */
/* /*
* If the object has been wiped upon free, make sure it's fully initialized by * If the object has been wiped upon free, make sure it's fully initialized by
@ -5041,7 +5028,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return NULL; return NULL;
if (empty) { if (empty) {
if (!refill_sheaf(s, empty, gfp)) { if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC)) {
full = empty; full = empty;
} else { } else {
/* /*
@ -5341,6 +5328,26 @@ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int nod
} }
EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
struct slab_sheaf *sheaf, gfp_t gfp)
{
int ret = 0;
ret = refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC);
if (likely(!ret || !gfp_pfmemalloc_allowed(gfp)))
return ret;
/*
* if we are allowed to, refill sheaf with pfmemalloc but then remember
* it for when it's returned
*/
ret = refill_sheaf(s, sheaf, gfp);
sheaf->pfmemalloc = true;
return ret;
}
/* /*
* returns a sheaf that has at least the requested size * returns a sheaf that has at least the requested size
* when prefilling is needed, do so with given gfp flags * when prefilling is needed, do so with given gfp flags
@ -5375,6 +5382,10 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
sheaf->cache = s; sheaf->cache = s;
sheaf->capacity = size; sheaf->capacity = size;
/*
* we do not need to care about pfmemalloc here because oversize
* sheaves area always flushed and freed when returned
*/
if (!__kmem_cache_alloc_bulk(s, gfp, size, if (!__kmem_cache_alloc_bulk(s, gfp, size,
&sheaf->objects[0])) { &sheaf->objects[0])) {
kfree(sheaf); kfree(sheaf);
@ -5411,17 +5422,18 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
if (!sheaf) if (!sheaf)
sheaf = alloc_empty_sheaf(s, gfp); sheaf = alloc_empty_sheaf(s, gfp);
if (sheaf && sheaf->size < size) { if (sheaf) {
if (refill_sheaf(s, sheaf, gfp)) { sheaf->capacity = s->sheaf_capacity;
sheaf->pfmemalloc = false;
if (sheaf->size < size &&
__prefill_sheaf_pfmemalloc(s, sheaf, gfp)) {
sheaf_flush_unused(s, sheaf); sheaf_flush_unused(s, sheaf);
free_empty_sheaf(s, sheaf); free_empty_sheaf(s, sheaf);
sheaf = NULL; sheaf = NULL;
} }
} }
if (sheaf)
sheaf->capacity = s->sheaf_capacity;
return sheaf; return sheaf;
} }
@ -5441,7 +5453,8 @@ void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
struct slub_percpu_sheaves *pcs; struct slub_percpu_sheaves *pcs;
struct node_barn *barn; struct node_barn *barn;
if (unlikely(sheaf->capacity != s->sheaf_capacity)) { if (unlikely((sheaf->capacity != s->sheaf_capacity)
|| sheaf->pfmemalloc)) {
sheaf_flush_unused(s, sheaf); sheaf_flush_unused(s, sheaf);
kfree(sheaf); kfree(sheaf);
return; return;
@ -5507,7 +5520,7 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
if (likely(sheaf->capacity >= size)) { if (likely(sheaf->capacity >= size)) {
if (likely(sheaf->capacity == s->sheaf_capacity)) if (likely(sheaf->capacity == s->sheaf_capacity))
return refill_sheaf(s, sheaf, gfp); return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size, if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
&sheaf->objects[sheaf->size])) { &sheaf->objects[sheaf->size])) {
@ -5540,6 +5553,9 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
* *
* The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT
* memcg charging is forced over limit if necessary, to avoid failure. * memcg charging is forced over limit if necessary, to avoid failure.
*
* It is possible that the allocation comes from kfence and then the sheaf
* size is not decreased.
*/ */
void * void *
kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
@ -5551,7 +5567,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
if (sheaf->size == 0) if (sheaf->size == 0)
goto out; goto out;
ret = sheaf->objects[--sheaf->size]; ret = kfence_alloc(s, s->object_size, gfp);
if (likely(!ret))
ret = sheaf->objects[--sheaf->size];
init = slab_want_init_on_alloc(gfp, s); init = slab_want_init_on_alloc(gfp, s);
@ -5719,9 +5738,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node)
* it did local_lock_irqsave(&s->cpu_slab->lock, flags). * it did local_lock_irqsave(&s->cpu_slab->lock, flags).
* In this case fast path with __update_cpu_freelist_fast() is not safe. * In this case fast path with __update_cpu_freelist_fast() is not safe.
*/ */
#ifndef CONFIG_SLUB_TINY
if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock))
#endif
ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size);
if (PTR_ERR(ret) == -EBUSY) { if (PTR_ERR(ret) == -EBUSY) {
@ -5859,8 +5876,8 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
unsigned long addr) unsigned long addr)
{ {
void *prior; void *old_head;
int was_frozen; bool was_frozen, was_full;
struct slab new; struct slab new;
unsigned long counters; unsigned long counters;
struct kmem_cache_node *n = NULL; struct kmem_cache_node *n = NULL;
@ -5874,20 +5891,37 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
return; return;
} }
/*
* It is enough to test IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) below
* instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s)
* is the only other reason it can be false, and it is already handled
* above.
*/
do { do {
if (unlikely(n)) { if (unlikely(n)) {
spin_unlock_irqrestore(&n->list_lock, flags); spin_unlock_irqrestore(&n->list_lock, flags);
n = NULL; n = NULL;
} }
prior = slab->freelist; old_head = slab->freelist;
counters = slab->counters; counters = slab->counters;
set_freepointer(s, tail, prior); set_freepointer(s, tail, old_head);
new.counters = counters; new.counters = counters;
was_frozen = new.frozen; was_frozen = !!new.frozen;
was_full = (old_head == NULL);
new.inuse -= cnt; new.inuse -= cnt;
if ((!new.inuse || !prior) && !was_frozen) { /*
/* Needs to be taken off a list */ * Might need to be taken off (due to becoming empty) or added
if (!kmem_cache_has_cpu_partial(s) || prior) { * to (due to not being full anymore) the partial list.
* Unless it's frozen.
*/
if ((!new.inuse || was_full) && !was_frozen) {
/*
* If slab becomes non-full and we have cpu partial
* lists, we put it there unconditionally to avoid
* taking the list_lock. Otherwise we need it.
*/
if (!(IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full)) {
n = get_node(s, slab_nid(slab)); n = get_node(s, slab_nid(slab));
/* /*
@ -5905,7 +5939,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
} }
} while (!slab_update_freelist(s, slab, } while (!slab_update_freelist(s, slab,
prior, counters, old_head, counters,
head, new.counters, head, new.counters,
"__slab_free")); "__slab_free"));
@ -5917,7 +5951,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
* activity can be necessary. * activity can be necessary.
*/ */
stat(s, FREE_FROZEN); stat(s, FREE_FROZEN);
} else if (kmem_cache_has_cpu_partial(s) && !prior) { } else if (IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full) {
/* /*
* If we started with a full slab then put it onto the * If we started with a full slab then put it onto the
* per cpu partial list. * per cpu partial list.
@ -5926,6 +5960,11 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
stat(s, CPU_PARTIAL_FREE); stat(s, CPU_PARTIAL_FREE);
} }
/*
* In other cases we didn't take the list_lock because the slab
* was already on the partial list and will remain there.
*/
return; return;
} }
@ -5933,19 +5972,24 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
* This slab was partially empty but not on the per-node partial list, * This slab was partially empty but not on the per-node partial list,
* in which case we shouldn't manipulate its list, just return. * in which case we shouldn't manipulate its list, just return.
*/ */
if (prior && !on_node_partial) { if (!was_full && !on_node_partial) {
spin_unlock_irqrestore(&n->list_lock, flags); spin_unlock_irqrestore(&n->list_lock, flags);
return; return;
} }
/*
* If slab became empty, should we add/keep it on the partial list or we
* have enough?
*/
if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
goto slab_empty; goto slab_empty;
/* /*
* Objects left in the slab. If it was not on the partial list before * Objects left in the slab. If it was not on the partial list before
* then add it. * then add it. This can only happen when cache has no per cpu partial
* list otherwise we would have put it there.
*/ */
if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
add_partial(n, slab, DEACTIVATE_TO_TAIL); add_partial(n, slab, DEACTIVATE_TO_TAIL);
stat(s, FREE_ADD_PARTIAL); stat(s, FREE_ADD_PARTIAL);
} }
@ -5953,10 +5997,11 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
return; return;
slab_empty: slab_empty:
if (prior) { /*
/* * The slab could have a single object and thus go from full to empty in
* Slab on the partial list. * a single free, but more likely it was on the partial list. Remove it.
*/ */
if (likely(!was_full)) {
remove_partial(n, slab); remove_partial(n, slab);
stat(s, FREE_REMOVE_PARTIAL); stat(s, FREE_REMOVE_PARTIAL);
} }
@ -6181,8 +6226,12 @@ static void rcu_free_sheaf(struct rcu_head *head)
* handles it fine. The only downside is that sheaf will serve fewer * handles it fine. The only downside is that sheaf will serve fewer
* allocations when reused. It only happens due to debugging, which is a * allocations when reused. It only happens due to debugging, which is a
* performance hit anyway. * performance hit anyway.
*
* If it returns true, there was at least one object from pfmemalloc
* slab so simply flush everything.
*/ */
__rcu_free_sheaf_prepare(s, sheaf); if (__rcu_free_sheaf_prepare(s, sheaf))
goto flush;
n = get_node(s, sheaf->node); n = get_node(s, sheaf->node);
if (!n) if (!n)
@ -6337,7 +6386,8 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
continue; continue;
} }
if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node)) { if (unlikely((IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node)
|| slab_test_pfmemalloc(slab))) {
remote_objects[remote_nr] = p[i]; remote_objects[remote_nr] = p[i];
p[i] = p[--size]; p[i] = p[--size];
if (++remote_nr >= PCS_BATCH_MAX) if (++remote_nr >= PCS_BATCH_MAX)
@ -6479,14 +6529,10 @@ static void free_deferred_objects(struct irq_work *work)
llist_for_each_safe(pos, t, llnode) { llist_for_each_safe(pos, t, llnode) {
struct slab *slab = container_of(pos, struct slab, llnode); struct slab *slab = container_of(pos, struct slab, llnode);
#ifdef CONFIG_SLUB_TINY
free_slab(slab->slab_cache, slab);
#else
if (slab->frozen) if (slab->frozen)
deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); deactivate_slab(slab->slab_cache, slab, slab->flush_freelist);
else else
free_slab(slab->slab_cache, slab); free_slab(slab->slab_cache, slab);
#endif
} }
} }
@ -6522,7 +6568,6 @@ void defer_free_barrier(void)
irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work);
} }
#ifndef CONFIG_SLUB_TINY
/* /*
* Fastpath with forced inlining to produce a kfree and kmem_cache_free that * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
* can perform fastpath freeing without additional function calls. * can perform fastpath freeing without additional function calls.
@ -6615,14 +6660,6 @@ static __always_inline void do_slab_free(struct kmem_cache *s,
} }
stat_add(s, FREE_FASTPATH, cnt); stat_add(s, FREE_FASTPATH, cnt);
} }
#else /* CONFIG_SLUB_TINY */
static void do_slab_free(struct kmem_cache *s,
struct slab *slab, void *head, void *tail,
int cnt, unsigned long addr)
{
__slab_free(s, slab, head, tail, cnt, addr);
}
#endif /* CONFIG_SLUB_TINY */
static __fastpath_inline static __fastpath_inline
void slab_free(struct kmem_cache *s, struct slab *slab, void *object, void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
@ -6635,7 +6672,8 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
return; return;
if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) || if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) ||
slab_nid(slab) == numa_mem_id())) { slab_nid(slab) == numa_mem_id())
&& likely(!slab_test_pfmemalloc(slab))) {
if (likely(free_to_pcs(s, object))) if (likely(free_to_pcs(s, object)))
return; return;
} }
@ -6899,11 +6937,7 @@ void kfree_nolock(const void *object)
* since kasan quarantine takes locks and not supported from NMI. * since kasan quarantine takes locks and not supported from NMI.
*/ */
kasan_slab_free(s, x, false, false, /* skip quarantine */true); kasan_slab_free(s, x, false, false, /* skip quarantine */true);
#ifndef CONFIG_SLUB_TINY
do_slab_free(s, slab, x, x, 0, _RET_IP_); do_slab_free(s, slab, x, x, 0, _RET_IP_);
#else
defer_free(s, x);
#endif
} }
EXPORT_SYMBOL_GPL(kfree_nolock); EXPORT_SYMBOL_GPL(kfree_nolock);
@ -7353,7 +7387,6 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
} }
EXPORT_SYMBOL(kmem_cache_free_bulk); EXPORT_SYMBOL(kmem_cache_free_bulk);
#ifndef CONFIG_SLUB_TINY
static inline static inline
int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
void **p) void **p)
@ -7371,14 +7404,8 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
local_lock_irqsave(&s->cpu_slab->lock, irqflags); local_lock_irqsave(&s->cpu_slab->lock, irqflags);
for (i = 0; i < size; i++) { for (i = 0; i < size; i++) {
void *object = kfence_alloc(s, s->object_size, flags); void *object = c->freelist;
if (unlikely(object)) {
p[i] = object;
continue;
}
object = c->freelist;
if (unlikely(!object)) { if (unlikely(!object)) {
/* /*
* We may have removed an object from c->freelist using * We may have removed an object from c->freelist using
@ -7424,41 +7451,13 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
return 0; return 0;
} }
#else /* CONFIG_SLUB_TINY */
static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p)
{
int i;
for (i = 0; i < size; i++) {
void *object = kfence_alloc(s, s->object_size, flags);
if (unlikely(object)) {
p[i] = object;
continue;
}
p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE,
_RET_IP_, s->object_size);
if (unlikely(!p[i]))
goto error;
maybe_wipe_obj_freeptr(s, p[i]);
}
return i;
error:
__kmem_cache_free_bulk(s, i, p);
return 0;
}
#endif /* CONFIG_SLUB_TINY */
/* Note that interrupts must be enabled when calling this function. */ /* Note that interrupts must be enabled when calling this function. */
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
void **p) void **p)
{ {
unsigned int i = 0; unsigned int i = 0;
void *kfence_obj;
if (!size) if (!size)
return 0; return 0;
@ -7467,6 +7466,20 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
if (unlikely(!s)) if (unlikely(!s))
return 0; return 0;
/*
* to make things simpler, only assume at most once kfence allocated
* object per bulk allocation and choose its index randomly
*/
kfence_obj = kfence_alloc(s, s->object_size, flags);
if (unlikely(kfence_obj)) {
if (unlikely(size == 1)) {
p[0] = kfence_obj;
goto out;
}
size--;
}
if (s->cpu_sheaves) if (s->cpu_sheaves)
i = alloc_from_pcs_bulk(s, size, p); i = alloc_from_pcs_bulk(s, size, p);
@ -7478,10 +7491,23 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) {
if (i > 0) if (i > 0)
__kmem_cache_free_bulk(s, i, p); __kmem_cache_free_bulk(s, i, p);
if (kfence_obj)
__kfence_free(kfence_obj);
return 0; return 0;
} }
} }
if (unlikely(kfence_obj)) {
int idx = get_random_u32_below(size + 1);
if (idx != size)
p[size] = p[idx];
p[idx] = kfence_obj;
size++;
}
out:
/* /*
* memcg and kmem_cache debug support and memory initialization. * memcg and kmem_cache debug support and memory initialization.
* Done outside of the IRQ disabled fastpath loop. * Done outside of the IRQ disabled fastpath loop.
@ -7643,7 +7669,6 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn)
barn_init(barn); barn_init(barn);
} }
#ifndef CONFIG_SLUB_TINY
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{ {
BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
@ -7664,12 +7689,6 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
return 1; return 1;
} }
#else
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
return 1;
}
#endif /* CONFIG_SLUB_TINY */
static int init_percpu_sheaves(struct kmem_cache *s) static int init_percpu_sheaves(struct kmem_cache *s)
{ {
@ -7759,13 +7778,11 @@ void __kmem_cache_release(struct kmem_cache *s)
cache_random_seq_destroy(s); cache_random_seq_destroy(s);
if (s->cpu_sheaves) if (s->cpu_sheaves)
pcs_destroy(s); pcs_destroy(s);
#ifndef CONFIG_SLUB_TINY
#ifdef CONFIG_PREEMPT_RT #ifdef CONFIG_PREEMPT_RT
if (s->cpu_slab) if (s->cpu_slab)
lockdep_unregister_key(&s->lock_key); lockdep_unregister_key(&s->lock_key);
#endif #endif
free_percpu(s->cpu_slab); free_percpu(s->cpu_slab);
#endif
free_kmem_cache_nodes(s); free_kmem_cache_nodes(s);
} }
@ -8519,10 +8536,8 @@ void __init kmem_cache_init(void)
void __init kmem_cache_init_late(void) void __init kmem_cache_init_late(void)
{ {
#ifndef CONFIG_SLUB_TINY
flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0);
WARN_ON(!flushwq); WARN_ON(!flushwq);
#endif
} }
struct kmem_cache * struct kmem_cache *