mirror of https://github.com/torvalds/linux.git
slab: Adjust placement of __kvmalloc_node_noprof
Move __kvmalloc_node_noprof (as well as kvfree*, kvrealloc_noprof and kmalloc_gfp_adjust for consistency) into mm/slub.c so that it can directly invoke __do_kmalloc_node, which is needed for the next patch. No functional changes intended. Signed-off-by: GONG Ruiqi <gongruiqi1@huawei.com> Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
This commit is contained in:
GONG Ruiqi
Vlastimil Babka
parent
12f4888c9d
commit
f1157db8b5
162
mm/slub.c
162
mm/slub.c
|
|
@ -4878,6 +4878,168 @@ void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(krealloc_noprof);
|
EXPORT_SYMBOL(krealloc_noprof);
|
||||||
|
|
||||||
|
static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* We want to attempt a large physically contiguous block first because
|
||||||
|
* it is less likely to fragment multiple larger blocks and therefore
|
||||||
|
* contribute to a long term fragmentation less than vmalloc fallback.
|
||||||
|
* However make sure that larger requests are not too disruptive - no
|
||||||
|
* OOM killer and no allocation failure warnings as we have a fallback.
|
||||||
|
*/
|
||||||
|
if (size > PAGE_SIZE) {
|
||||||
|
flags |= __GFP_NOWARN;
|
||||||
|
|
||||||
|
if (!(flags & __GFP_RETRY_MAYFAIL))
|
||||||
|
flags |= __GFP_NORETRY;
|
||||||
|
|
||||||
|
/* nofail semantic is implemented by the vmalloc fallback */
|
||||||
|
flags &= ~__GFP_NOFAIL;
|
||||||
|
}
|
||||||
|
|
||||||
|
return flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* __kvmalloc_node - attempt to allocate physically contiguous memory, but upon
|
||||||
|
* failure, fall back to non-contiguous (vmalloc) allocation.
|
||||||
|
* @size: size of the request.
|
||||||
|
* @b: which set of kmalloc buckets to allocate from.
|
||||||
|
* @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
|
||||||
|
* @node: numa node to allocate from
|
||||||
|
*
|
||||||
|
* Uses kmalloc to get the memory but if the allocation fails then falls back
|
||||||
|
* to the vmalloc allocator. Use kvfree for freeing the memory.
|
||||||
|
*
|
||||||
|
* GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier.
|
||||||
|
* __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
|
||||||
|
* preferable to the vmalloc fallback, due to visible performance drawbacks.
|
||||||
|
*
|
||||||
|
* Return: pointer to the allocated memory of %NULL in case of failure
|
||||||
|
*/
|
||||||
|
void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
|
||||||
|
{
|
||||||
|
void *ret;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* It doesn't really make sense to fallback to vmalloc for sub page
|
||||||
|
* requests
|
||||||
|
*/
|
||||||
|
ret = __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, b),
|
||||||
|
kmalloc_gfp_adjust(flags, size),
|
||||||
|
node);
|
||||||
|
if (ret || size <= PAGE_SIZE)
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
/* non-sleeping allocations are not supported by vmalloc */
|
||||||
|
if (!gfpflags_allow_blocking(flags))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
/* Don't even allow crazy sizes */
|
||||||
|
if (unlikely(size > INT_MAX)) {
|
||||||
|
WARN_ON_ONCE(!(flags & __GFP_NOWARN));
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
|
||||||
|
* since the callers already cannot assume anything
|
||||||
|
* about the resulting pointer, and cannot play
|
||||||
|
* protection games.
|
||||||
|
*/
|
||||||
|
return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
|
||||||
|
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
|
||||||
|
node, __builtin_return_address(0));
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(__kvmalloc_node_noprof);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* kvfree() - Free memory.
|
||||||
|
* @addr: Pointer to allocated memory.
|
||||||
|
*
|
||||||
|
* kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
|
||||||
|
* It is slightly more efficient to use kfree() or vfree() if you are certain
|
||||||
|
* that you know which one to use.
|
||||||
|
*
|
||||||
|
* Context: Either preemptible task context or not-NMI interrupt.
|
||||||
|
*/
|
||||||
|
void kvfree(const void *addr)
|
||||||
|
{
|
||||||
|
if (is_vmalloc_addr(addr))
|
||||||
|
vfree(addr);
|
||||||
|
else
|
||||||
|
kfree(addr);
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(kvfree);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* kvfree_sensitive - Free a data object containing sensitive information.
|
||||||
|
* @addr: address of the data object to be freed.
|
||||||
|
* @len: length of the data object.
|
||||||
|
*
|
||||||
|
* Use the special memzero_explicit() function to clear the content of a
|
||||||
|
* kvmalloc'ed object containing sensitive data to make sure that the
|
||||||
|
* compiler won't optimize out the data clearing.
|
||||||
|
*/
|
||||||
|
void kvfree_sensitive(const void *addr, size_t len)
|
||||||
|
{
|
||||||
|
if (likely(!ZERO_OR_NULL_PTR(addr))) {
|
||||||
|
memzero_explicit((void *)addr, len);
|
||||||
|
kvfree(addr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(kvfree_sensitive);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* kvrealloc - reallocate memory; contents remain unchanged
|
||||||
|
* @p: object to reallocate memory for
|
||||||
|
* @size: the size to reallocate
|
||||||
|
* @flags: the flags for the page level allocator
|
||||||
|
*
|
||||||
|
* If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
|
||||||
|
* and @p is not a %NULL pointer, the object pointed to is freed.
|
||||||
|
*
|
||||||
|
* If __GFP_ZERO logic is requested, callers must ensure that, starting with the
|
||||||
|
* initial memory allocation, every subsequent call to this API for the same
|
||||||
|
* memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
|
||||||
|
* __GFP_ZERO is not fully honored by this API.
|
||||||
|
*
|
||||||
|
* In any case, the contents of the object pointed to are preserved up to the
|
||||||
|
* lesser of the new and old sizes.
|
||||||
|
*
|
||||||
|
* This function must not be called concurrently with itself or kvfree() for the
|
||||||
|
* same memory allocation.
|
||||||
|
*
|
||||||
|
* Return: pointer to the allocated memory or %NULL in case of error
|
||||||
|
*/
|
||||||
|
void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)
|
||||||
|
{
|
||||||
|
void *n;
|
||||||
|
|
||||||
|
if (is_vmalloc_addr(p))
|
||||||
|
return vrealloc_noprof(p, size, flags);
|
||||||
|
|
||||||
|
n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size));
|
||||||
|
if (!n) {
|
||||||
|
/* We failed to krealloc(), fall back to kvmalloc(). */
|
||||||
|
n = kvmalloc_noprof(size, flags);
|
||||||
|
if (!n)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
if (p) {
|
||||||
|
/* We already know that `p` is not a vmalloc address. */
|
||||||
|
kasan_disable_current();
|
||||||
|
memcpy(n, kasan_reset_tag(p), ksize(p));
|
||||||
|
kasan_enable_current();
|
||||||
|
|
||||||
|
kfree(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(kvrealloc_noprof);
|
||||||
|
|
||||||
struct detached_freelist {
|
struct detached_freelist {
|
||||||
struct slab *slab;
|
struct slab *slab;
|
||||||
void *tail;
|
void *tail;
|
||||||
|
|
|
||||||
162
mm/util.c
162
mm/util.c
|
|
@ -612,168 +612,6 @@ unsigned long vm_mmap(struct file *file, unsigned long addr,
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(vm_mmap);
|
EXPORT_SYMBOL(vm_mmap);
|
||||||
|
|
||||||
static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
|
|
||||||
{
|
|
||||||
/*
|
|
||||||
* We want to attempt a large physically contiguous block first because
|
|
||||||
* it is less likely to fragment multiple larger blocks and therefore
|
|
||||||
* contribute to a long term fragmentation less than vmalloc fallback.
|
|
||||||
* However make sure that larger requests are not too disruptive - no
|
|
||||||
* OOM killer and no allocation failure warnings as we have a fallback.
|
|
||||||
*/
|
|
||||||
if (size > PAGE_SIZE) {
|
|
||||||
flags |= __GFP_NOWARN;
|
|
||||||
|
|
||||||
if (!(flags & __GFP_RETRY_MAYFAIL))
|
|
||||||
flags |= __GFP_NORETRY;
|
|
||||||
|
|
||||||
/* nofail semantic is implemented by the vmalloc fallback */
|
|
||||||
flags &= ~__GFP_NOFAIL;
|
|
||||||
}
|
|
||||||
|
|
||||||
return flags;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* __kvmalloc_node - attempt to allocate physically contiguous memory, but upon
|
|
||||||
* failure, fall back to non-contiguous (vmalloc) allocation.
|
|
||||||
* @size: size of the request.
|
|
||||||
* @b: which set of kmalloc buckets to allocate from.
|
|
||||||
* @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
|
|
||||||
* @node: numa node to allocate from
|
|
||||||
*
|
|
||||||
* Uses kmalloc to get the memory but if the allocation fails then falls back
|
|
||||||
* to the vmalloc allocator. Use kvfree for freeing the memory.
|
|
||||||
*
|
|
||||||
* GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier.
|
|
||||||
* __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
|
|
||||||
* preferable to the vmalloc fallback, due to visible performance drawbacks.
|
|
||||||
*
|
|
||||||
* Return: pointer to the allocated memory of %NULL in case of failure
|
|
||||||
*/
|
|
||||||
void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
|
|
||||||
{
|
|
||||||
void *ret;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* It doesn't really make sense to fallback to vmalloc for sub page
|
|
||||||
* requests
|
|
||||||
*/
|
|
||||||
ret = __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, b),
|
|
||||||
kmalloc_gfp_adjust(flags, size),
|
|
||||||
node);
|
|
||||||
if (ret || size <= PAGE_SIZE)
|
|
||||||
return ret;
|
|
||||||
|
|
||||||
/* non-sleeping allocations are not supported by vmalloc */
|
|
||||||
if (!gfpflags_allow_blocking(flags))
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
/* Don't even allow crazy sizes */
|
|
||||||
if (unlikely(size > INT_MAX)) {
|
|
||||||
WARN_ON_ONCE(!(flags & __GFP_NOWARN));
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
|
|
||||||
* since the callers already cannot assume anything
|
|
||||||
* about the resulting pointer, and cannot play
|
|
||||||
* protection games.
|
|
||||||
*/
|
|
||||||
return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
|
|
||||||
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
|
|
||||||
node, __builtin_return_address(0));
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL(__kvmalloc_node_noprof);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* kvfree() - Free memory.
|
|
||||||
* @addr: Pointer to allocated memory.
|
|
||||||
*
|
|
||||||
* kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
|
|
||||||
* It is slightly more efficient to use kfree() or vfree() if you are certain
|
|
||||||
* that you know which one to use.
|
|
||||||
*
|
|
||||||
* Context: Either preemptible task context or not-NMI interrupt.
|
|
||||||
*/
|
|
||||||
void kvfree(const void *addr)
|
|
||||||
{
|
|
||||||
if (is_vmalloc_addr(addr))
|
|
||||||
vfree(addr);
|
|
||||||
else
|
|
||||||
kfree(addr);
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL(kvfree);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* kvfree_sensitive - Free a data object containing sensitive information.
|
|
||||||
* @addr: address of the data object to be freed.
|
|
||||||
* @len: length of the data object.
|
|
||||||
*
|
|
||||||
* Use the special memzero_explicit() function to clear the content of a
|
|
||||||
* kvmalloc'ed object containing sensitive data to make sure that the
|
|
||||||
* compiler won't optimize out the data clearing.
|
|
||||||
*/
|
|
||||||
void kvfree_sensitive(const void *addr, size_t len)
|
|
||||||
{
|
|
||||||
if (likely(!ZERO_OR_NULL_PTR(addr))) {
|
|
||||||
memzero_explicit((void *)addr, len);
|
|
||||||
kvfree(addr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL(kvfree_sensitive);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* kvrealloc - reallocate memory; contents remain unchanged
|
|
||||||
* @p: object to reallocate memory for
|
|
||||||
* @size: the size to reallocate
|
|
||||||
* @flags: the flags for the page level allocator
|
|
||||||
*
|
|
||||||
* If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
|
|
||||||
* and @p is not a %NULL pointer, the object pointed to is freed.
|
|
||||||
*
|
|
||||||
* If __GFP_ZERO logic is requested, callers must ensure that, starting with the
|
|
||||||
* initial memory allocation, every subsequent call to this API for the same
|
|
||||||
* memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
|
|
||||||
* __GFP_ZERO is not fully honored by this API.
|
|
||||||
*
|
|
||||||
* In any case, the contents of the object pointed to are preserved up to the
|
|
||||||
* lesser of the new and old sizes.
|
|
||||||
*
|
|
||||||
* This function must not be called concurrently with itself or kvfree() for the
|
|
||||||
* same memory allocation.
|
|
||||||
*
|
|
||||||
* Return: pointer to the allocated memory or %NULL in case of error
|
|
||||||
*/
|
|
||||||
void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)
|
|
||||||
{
|
|
||||||
void *n;
|
|
||||||
|
|
||||||
if (is_vmalloc_addr(p))
|
|
||||||
return vrealloc_noprof(p, size, flags);
|
|
||||||
|
|
||||||
n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size));
|
|
||||||
if (!n) {
|
|
||||||
/* We failed to krealloc(), fall back to kvmalloc(). */
|
|
||||||
n = kvmalloc_noprof(size, flags);
|
|
||||||
if (!n)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
if (p) {
|
|
||||||
/* We already know that `p` is not a vmalloc address. */
|
|
||||||
kasan_disable_current();
|
|
||||||
memcpy(n, kasan_reset_tag(p), ksize(p));
|
|
||||||
kasan_enable_current();
|
|
||||||
|
|
||||||
kfree(p);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return n;
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL(kvrealloc_noprof);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* __vmalloc_array - allocate memory for a virtually contiguous array.
|
* __vmalloc_array - allocate memory for a virtually contiguous array.
|
||||||
* @n: number of elements.
|
* @n: number of elements.
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue