mirror of https://github.com/torvalds/linux.git
slab: Adjust placement of __kvmalloc_node_noprof
Move __kvmalloc_node_noprof (as well as kvfree*, kvrealloc_noprof and kmalloc_gfp_adjust for consistency) into mm/slub.c so that it can directly invoke __do_kmalloc_node, which is needed for the next patch. No functional changes intended. Signed-off-by: GONG Ruiqi <gongruiqi1@huawei.com> Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
This commit is contained in:
GONG Ruiqi
Vlastimil Babka
parent
12f4888c9d
commit
f1157db8b5
162
mm/slub.c
162
mm/slub.c
|
|
@ -4878,6 +4878,168 @@ void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)
|
|||
}
|
||||
EXPORT_SYMBOL(krealloc_noprof);
|
||||
|
||||
static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
|
||||
{
|
||||
/*
|
||||
* We want to attempt a large physically contiguous block first because
|
||||
* it is less likely to fragment multiple larger blocks and therefore
|
||||
* contribute to a long term fragmentation less than vmalloc fallback.
|
||||
* However make sure that larger requests are not too disruptive - no
|
||||
* OOM killer and no allocation failure warnings as we have a fallback.
|
||||
*/
|
||||
if (size > PAGE_SIZE) {
|
||||
flags |= __GFP_NOWARN;
|
||||
|
||||
if (!(flags & __GFP_RETRY_MAYFAIL))
|
||||
flags |= __GFP_NORETRY;
|
||||
|
||||
/* nofail semantic is implemented by the vmalloc fallback */
|
||||
flags &= ~__GFP_NOFAIL;
|
||||
}
|
||||
|
||||
return flags;
|
||||
}
|
||||
|
||||
/**
|
||||
* __kvmalloc_node - attempt to allocate physically contiguous memory, but upon
|
||||
* failure, fall back to non-contiguous (vmalloc) allocation.
|
||||
* @size: size of the request.
|
||||
* @b: which set of kmalloc buckets to allocate from.
|
||||
* @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
|
||||
* @node: numa node to allocate from
|
||||
*
|
||||
* Uses kmalloc to get the memory but if the allocation fails then falls back
|
||||
* to the vmalloc allocator. Use kvfree for freeing the memory.
|
||||
*
|
||||
* GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier.
|
||||
* __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
|
||||
* preferable to the vmalloc fallback, due to visible performance drawbacks.
|
||||
*
|
||||
* Return: pointer to the allocated memory of %NULL in case of failure
|
||||
*/
|
||||
void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
/*
|
||||
* It doesn't really make sense to fallback to vmalloc for sub page
|
||||
* requests
|
||||
*/
|
||||
ret = __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, b),
|
||||
kmalloc_gfp_adjust(flags, size),
|
||||
node);
|
||||
if (ret || size <= PAGE_SIZE)
|
||||
return ret;
|
||||
|
||||
/* non-sleeping allocations are not supported by vmalloc */
|
||||
if (!gfpflags_allow_blocking(flags))
|
||||
return NULL;
|
||||
|
||||
/* Don't even allow crazy sizes */
|
||||
if (unlikely(size > INT_MAX)) {
|
||||
WARN_ON_ONCE(!(flags & __GFP_NOWARN));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
|
||||
* since the callers already cannot assume anything
|
||||
* about the resulting pointer, and cannot play
|
||||
* protection games.
|
||||
*/
|
||||
return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
|
||||
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
|
||||
node, __builtin_return_address(0));
|
||||
}
|
||||
EXPORT_SYMBOL(__kvmalloc_node_noprof);
|
||||
|
||||
/**
|
||||
* kvfree() - Free memory.
|
||||
* @addr: Pointer to allocated memory.
|
||||
*
|
||||
* kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
|
||||
* It is slightly more efficient to use kfree() or vfree() if you are certain
|
||||
* that you know which one to use.
|
||||
*
|
||||
* Context: Either preemptible task context or not-NMI interrupt.
|
||||
*/
|
||||
void kvfree(const void *addr)
|
||||
{
|
||||
if (is_vmalloc_addr(addr))
|
||||
vfree(addr);
|
||||
else
|
||||
kfree(addr);
|
||||
}
|
||||
EXPORT_SYMBOL(kvfree);
|
||||
|
||||
/**
|
||||
* kvfree_sensitive - Free a data object containing sensitive information.
|
||||
* @addr: address of the data object to be freed.
|
||||
* @len: length of the data object.
|
||||
*
|
||||
* Use the special memzero_explicit() function to clear the content of a
|
||||
* kvmalloc'ed object containing sensitive data to make sure that the
|
||||
* compiler won't optimize out the data clearing.
|
||||
*/
|
||||
void kvfree_sensitive(const void *addr, size_t len)
|
||||
{
|
||||
if (likely(!ZERO_OR_NULL_PTR(addr))) {
|
||||
memzero_explicit((void *)addr, len);
|
||||
kvfree(addr);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL(kvfree_sensitive);
|
||||
|
||||
/**
|
||||
* kvrealloc - reallocate memory; contents remain unchanged
|
||||
* @p: object to reallocate memory for
|
||||
* @size: the size to reallocate
|
||||
* @flags: the flags for the page level allocator
|
||||
*
|
||||
* If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
|
||||
* and @p is not a %NULL pointer, the object pointed to is freed.
|
||||
*
|
||||
* If __GFP_ZERO logic is requested, callers must ensure that, starting with the
|
||||
* initial memory allocation, every subsequent call to this API for the same
|
||||
* memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
|
||||
* __GFP_ZERO is not fully honored by this API.
|
||||
*
|
||||
* In any case, the contents of the object pointed to are preserved up to the
|
||||
* lesser of the new and old sizes.
|
||||
*
|
||||
* This function must not be called concurrently with itself or kvfree() for the
|
||||
* same memory allocation.
|
||||
*
|
||||
* Return: pointer to the allocated memory or %NULL in case of error
|
||||
*/
|
||||
void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)
|
||||
{
|
||||
void *n;
|
||||
|
||||
if (is_vmalloc_addr(p))
|
||||
return vrealloc_noprof(p, size, flags);
|
||||
|
||||
n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size));
|
||||
if (!n) {
|
||||
/* We failed to krealloc(), fall back to kvmalloc(). */
|
||||
n = kvmalloc_noprof(size, flags);
|
||||
if (!n)
|
||||
return NULL;
|
||||
|
||||
if (p) {
|
||||
/* We already know that `p` is not a vmalloc address. */
|
||||
kasan_disable_current();
|
||||
memcpy(n, kasan_reset_tag(p), ksize(p));
|
||||
kasan_enable_current();
|
||||
|
||||
kfree(p);
|
||||
}
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
EXPORT_SYMBOL(kvrealloc_noprof);
|
||||
|
||||
struct detached_freelist {
|
||||
struct slab *slab;
|
||||
void *tail;
|
||||
|
|
|
|||
162
mm/util.c
162
mm/util.c
|
|
@ -612,168 +612,6 @@ unsigned long vm_mmap(struct file *file, unsigned long addr,
|
|||
}
|
||||
EXPORT_SYMBOL(vm_mmap);
|
||||
|
||||
static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
|
||||
{
|
||||
/*
|
||||
* We want to attempt a large physically contiguous block first because
|
||||
* it is less likely to fragment multiple larger blocks and therefore
|
||||
* contribute to a long term fragmentation less than vmalloc fallback.
|
||||
* However make sure that larger requests are not too disruptive - no
|
||||
* OOM killer and no allocation failure warnings as we have a fallback.
|
||||
*/
|
||||
if (size > PAGE_SIZE) {
|
||||
flags |= __GFP_NOWARN;
|
||||
|
||||
if (!(flags & __GFP_RETRY_MAYFAIL))
|
||||
flags |= __GFP_NORETRY;
|
||||
|
||||
/* nofail semantic is implemented by the vmalloc fallback */
|
||||
flags &= ~__GFP_NOFAIL;
|
||||
}
|
||||
|
||||
return flags;
|
||||
}
|
||||
|
||||
/**
|
||||
* __kvmalloc_node - attempt to allocate physically contiguous memory, but upon
|
||||
* failure, fall back to non-contiguous (vmalloc) allocation.
|
||||
* @size: size of the request.
|
||||
* @b: which set of kmalloc buckets to allocate from.
|
||||
* @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
|
||||
* @node: numa node to allocate from
|
||||
*
|
||||
* Uses kmalloc to get the memory but if the allocation fails then falls back
|
||||
* to the vmalloc allocator. Use kvfree for freeing the memory.
|
||||
*
|
||||
* GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier.
|
||||
* __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
|
||||
* preferable to the vmalloc fallback, due to visible performance drawbacks.
|
||||
*
|
||||
* Return: pointer to the allocated memory of %NULL in case of failure
|
||||
*/
|
||||
void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
/*
|
||||
* It doesn't really make sense to fallback to vmalloc for sub page
|
||||
* requests
|
||||
*/
|
||||
ret = __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, b),
|
||||
kmalloc_gfp_adjust(flags, size),
|
||||
node);
|
||||
if (ret || size <= PAGE_SIZE)
|
||||
return ret;
|
||||
|
||||
/* non-sleeping allocations are not supported by vmalloc */
|
||||
if (!gfpflags_allow_blocking(flags))
|
||||
return NULL;
|
||||
|
||||
/* Don't even allow crazy sizes */
|
||||
if (unlikely(size > INT_MAX)) {
|
||||
WARN_ON_ONCE(!(flags & __GFP_NOWARN));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
|
||||
* since the callers already cannot assume anything
|
||||
* about the resulting pointer, and cannot play
|
||||
* protection games.
|
||||
*/
|
||||
return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
|
||||
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
|
||||
node, __builtin_return_address(0));
|
||||
}
|
||||
EXPORT_SYMBOL(__kvmalloc_node_noprof);
|
||||
|
||||
/**
|
||||
* kvfree() - Free memory.
|
||||
* @addr: Pointer to allocated memory.
|
||||
*
|
||||
* kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
|
||||
* It is slightly more efficient to use kfree() or vfree() if you are certain
|
||||
* that you know which one to use.
|
||||
*
|
||||
* Context: Either preemptible task context or not-NMI interrupt.
|
||||
*/
|
||||
void kvfree(const void *addr)
|
||||
{
|
||||
if (is_vmalloc_addr(addr))
|
||||
vfree(addr);
|
||||
else
|
||||
kfree(addr);
|
||||
}
|
||||
EXPORT_SYMBOL(kvfree);
|
||||
|
||||
/**
|
||||
* kvfree_sensitive - Free a data object containing sensitive information.
|
||||
* @addr: address of the data object to be freed.
|
||||
* @len: length of the data object.
|
||||
*
|
||||
* Use the special memzero_explicit() function to clear the content of a
|
||||
* kvmalloc'ed object containing sensitive data to make sure that the
|
||||
* compiler won't optimize out the data clearing.
|
||||
*/
|
||||
void kvfree_sensitive(const void *addr, size_t len)
|
||||
{
|
||||
if (likely(!ZERO_OR_NULL_PTR(addr))) {
|
||||
memzero_explicit((void *)addr, len);
|
||||
kvfree(addr);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL(kvfree_sensitive);
|
||||
|
||||
/**
|
||||
* kvrealloc - reallocate memory; contents remain unchanged
|
||||
* @p: object to reallocate memory for
|
||||
* @size: the size to reallocate
|
||||
* @flags: the flags for the page level allocator
|
||||
*
|
||||
* If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
|
||||
* and @p is not a %NULL pointer, the object pointed to is freed.
|
||||
*
|
||||
* If __GFP_ZERO logic is requested, callers must ensure that, starting with the
|
||||
* initial memory allocation, every subsequent call to this API for the same
|
||||
* memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
|
||||
* __GFP_ZERO is not fully honored by this API.
|
||||
*
|
||||
* In any case, the contents of the object pointed to are preserved up to the
|
||||
* lesser of the new and old sizes.
|
||||
*
|
||||
* This function must not be called concurrently with itself or kvfree() for the
|
||||
* same memory allocation.
|
||||
*
|
||||
* Return: pointer to the allocated memory or %NULL in case of error
|
||||
*/
|
||||
void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags)
|
||||
{
|
||||
void *n;
|
||||
|
||||
if (is_vmalloc_addr(p))
|
||||
return vrealloc_noprof(p, size, flags);
|
||||
|
||||
n = krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size));
|
||||
if (!n) {
|
||||
/* We failed to krealloc(), fall back to kvmalloc(). */
|
||||
n = kvmalloc_noprof(size, flags);
|
||||
if (!n)
|
||||
return NULL;
|
||||
|
||||
if (p) {
|
||||
/* We already know that `p` is not a vmalloc address. */
|
||||
kasan_disable_current();
|
||||
memcpy(n, kasan_reset_tag(p), ksize(p));
|
||||
kasan_enable_current();
|
||||
|
||||
kfree(p);
|
||||
}
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
EXPORT_SYMBOL(kvrealloc_noprof);
|
||||
|
||||
/**
|
||||
* __vmalloc_array - allocate memory for a virtually contiguous array.
|
||||
* @n: number of elements.
|
||||
|
|
|
|||
Loading…
Reference in New Issue