Skip to content

Commit

Permalink
mm/sl[au]b: generalize kmalloc subsystem
Browse files Browse the repository at this point in the history 
Now everything in kmalloc subsystem can be generalized.
Let's do it!

Generalize __do_kmalloc_node(), __kmalloc_node_track_caller(),
kfree(), __ksize(), __kmalloc(), __kmalloc_node() and move them
to slab_common.c.

In the meantime, rename kmalloc_large_node_notrace()
to __kmalloc_large_node() and make it static as it's now only called in
slab_common.c.

[ feng.tang@intel.com: adjust kfence skip list to include
  __kmem_cache_free so that kfence kunit tests do not fail ]

Signed-off-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
  • Loading branch information
@hygoni @tehcaster
hygoni authored and tehcaster committed Sep 1, 2022
1 parent ed4cd17 commit b140513
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 200 deletions.
1 change: 1 addition & 0 deletions mm/kfence/report.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries
/* Also the *_bulk() variants by only checking prefixes. */
if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))
goto found;
Expand Down
108 changes: 0 additions & 108 deletions mm/slab.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3587,44 +3587,6 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
#endif

static __always_inline void *
__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
{
struct kmem_cache *cachep;
void *ret;

if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
ret = kmalloc_large_node_notrace(size, flags, node);

trace_kmalloc_node(caller, ret, NULL, size,
PAGE_SIZE << get_order(size),
flags, node);
return ret;
}

cachep = kmalloc_slab(size, flags);
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
return cachep;

ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
ret = kasan_kmalloc(cachep, ret, size, flags);

return ret;
}

void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
return __do_kmalloc_node(size, flags, node, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc_node);

void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
int node, unsigned long caller)
{
return __do_kmalloc_node(size, flags, node, caller);
}
EXPORT_SYMBOL(__kmalloc_node_track_caller);

#ifdef CONFIG_PRINTK
void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
{
Expand All @@ -3647,12 +3609,6 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
}
#endif

void *__kmalloc(size_t size, gfp_t flags)
{
return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc);

static __always_inline
void __do_kmem_cache_free(struct kmem_cache *cachep, void *objp,
unsigned long caller)
Expand Down Expand Up @@ -3730,43 +3686,6 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

/**
* kfree - free previously allocated memory
* @objp: pointer returned by kmalloc.
*
* If @objp is NULL, no operation is performed.
*
* Don't free memory not originally allocated by kmalloc()
* or you will run into trouble.
*/
void kfree(const void *objp)
{
struct kmem_cache *c;
unsigned long flags;
struct folio *folio;

trace_kfree(_RET_IP_, objp);

if (unlikely(ZERO_OR_NULL_PTR(objp)))
return;

folio = virt_to_folio(objp);
if (!folio_test_slab(folio)) {
free_large_kmalloc(folio, (void *)objp);
return;
}

c = folio_slab(folio)->slab_cache;

local_irq_save(flags);
kfree_debugcheck(objp);
debug_check_no_locks_freed(objp, c->object_size);
debug_check_no_obj_freed(objp, c->object_size);
__cache_free(c, (void *)objp, _RET_IP_);
local_irq_restore(flags);
}
EXPORT_SYMBOL(kfree);

/*
* This initializes kmem_cache_node or resizes various caches for all nodes.
*/
Expand Down Expand Up @@ -4169,30 +4088,3 @@ void __check_heap_object(const void *ptr, unsigned long n,
usercopy_abort("SLAB object", cachep->name, to_user, offset, n);
}
#endif /* CONFIG_HARDENED_USERCOPY */

/**
* __ksize -- Uninstrumented ksize.
* @objp: pointer to the object
*
* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
* safety checks as ksize() with KASAN instrumentation enabled.
*
* Return: size of the actual memory used by @objp in bytes
*/
size_t __ksize(const void *objp)
{
struct kmem_cache *c;
struct folio *folio;

BUG_ON(!objp);
if (unlikely(objp == ZERO_SIZE_PTR))
return 0;

folio = virt_to_folio(objp);
if (!folio_test_slab(folio))
return folio_size(folio);

c = folio_slab(folio)->slab_cache;
return c->object_size;
}
EXPORT_SYMBOL(__ksize);
2 changes: 0 additions & 2 deletions mm/slab.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,6 @@ void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,
void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller);
#endif

void *kmalloc_large_node_notrace(size_t size, gfp_t flags, int node);

gfp_t kmalloc_fix_flags(gfp_t flags);

/* Functions provided by the slab allocators */
Expand Down
109 changes: 106 additions & 3 deletions mm/slab_common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,109 @@ void free_large_kmalloc(struct folio *folio, void *object)
-(PAGE_SIZE << order));
__free_pages(folio_page(folio, 0), order);
}

static void *__kmalloc_large_node(size_t size, gfp_t flags, int node);
static __always_inline
void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
{
struct kmem_cache *s;
void *ret;

if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
ret = __kmalloc_large_node(size, flags, node);
trace_kmalloc_node(caller, ret, NULL,
size, PAGE_SIZE << get_order(size),
flags, node);
return ret;
}

s = kmalloc_slab(size, flags);

if (unlikely(ZERO_OR_NULL_PTR(s)))
return s;

ret = __kmem_cache_alloc_node(s, flags, node, size, caller);
ret = kasan_kmalloc(s, ret, size, flags);
trace_kmalloc_node(caller, ret, s, size,
s->size, flags, node);
return ret;
}

void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
return __do_kmalloc_node(size, flags, node, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc_node);

void *__kmalloc(size_t size, gfp_t flags)
{
return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc);

void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
int node, unsigned long caller)
{
return __do_kmalloc_node(size, flags, node, caller);
}
EXPORT_SYMBOL(__kmalloc_node_track_caller);

/**
* kfree - free previously allocated memory
* @object: pointer returned by kmalloc.
*
* If @object is NULL, no operation is performed.
*
* Don't free memory not originally allocated by kmalloc()
* or you will run into trouble.
*/
void kfree(const void *object)
{
struct folio *folio;
struct slab *slab;
struct kmem_cache *s;

trace_kfree(_RET_IP_, object);

if (unlikely(ZERO_OR_NULL_PTR(object)))
return;

folio = virt_to_folio(object);
if (unlikely(!folio_test_slab(folio))) {
free_large_kmalloc(folio, (void *)object);
return;
}

slab = folio_slab(folio);
s = slab->slab_cache;
__kmem_cache_free(s, (void *)object, _RET_IP_);
}
EXPORT_SYMBOL(kfree);

/**
* __ksize -- Uninstrumented ksize.
* @object: pointer to the object
*
* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
* safety checks as ksize() with KASAN instrumentation enabled.
*
* Return: size of the actual memory used by @object in bytes
*/
size_t __ksize(const void *object)
{
struct folio *folio;

if (unlikely(object == ZERO_SIZE_PTR))
return 0;

folio = virt_to_folio(object);

if (unlikely(!folio_test_slab(folio)))
return folio_size(folio);

return slab_ksize(folio_slab(folio)->slab_cache);
}
EXPORT_SYMBOL(__ksize);
#endif /* !CONFIG_SLOB */

gfp_t kmalloc_fix_flags(gfp_t flags)
Expand All @@ -917,7 +1020,7 @@ gfp_t kmalloc_fix_flags(gfp_t flags)
* know the allocation order to free the pages properly in kfree.
*/

void *kmalloc_large_node_notrace(size_t size, gfp_t flags, int node)
static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
{
struct page *page;
void *ptr = NULL;
Expand All @@ -943,7 +1046,7 @@ void *kmalloc_large_node_notrace(size_t size, gfp_t flags, int node)

void *kmalloc_large(size_t size, gfp_t flags)
{
void *ret = kmalloc_large_node_notrace(size, flags, NUMA_NO_NODE);
void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);

trace_kmalloc(_RET_IP_, ret, NULL, size,
PAGE_SIZE << get_order(size), flags);
Expand All @@ -953,7 +1056,7 @@ EXPORT_SYMBOL(kmalloc_large);

void *kmalloc_large_node(size_t size, gfp_t flags, int node)
{
void *ret = kmalloc_large_node_notrace(size, flags, node);
void *ret = __kmalloc_large_node(size, flags, node);

trace_kmalloc_node(_RET_IP_, ret, NULL, size,
PAGE_SIZE << get_order(size), flags, node);
Expand Down
87 changes: 0 additions & 87 deletions mm/slub.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4388,49 +4388,6 @@ static int __init setup_slub_min_objects(char *str)

__setup("slub_min_objects=", setup_slub_min_objects);

static __always_inline
void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
{
struct kmem_cache *s;
void *ret;

if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
ret = kmalloc_large_node_notrace(size, flags, node);

trace_kmalloc_node(caller, ret, NULL,
size, PAGE_SIZE << get_order(size),
flags, node);

return ret;
}

s = kmalloc_slab(size, flags);

if (unlikely(ZERO_OR_NULL_PTR(s)))
return s;

ret = slab_alloc_node(s, NULL, flags, node, caller, size);

trace_kmalloc_node(caller, ret, s, size, s->size, flags, node);

ret = kasan_kmalloc(s, ret, size, flags);

return ret;
}

void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
return __do_kmalloc_node(size, flags, node, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc_node);

void *__kmalloc(size_t size, gfp_t flags)
{
return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc);


#ifdef CONFIG_HARDENED_USERCOPY
/*
* Rejects incorrectly sized objects and objects that are to be copied
Expand Down Expand Up @@ -4481,43 +4438,6 @@ void __check_heap_object(const void *ptr, unsigned long n,
}
#endif /* CONFIG_HARDENED_USERCOPY */

size_t __ksize(const void *object)
{
struct folio *folio;

if (unlikely(object == ZERO_SIZE_PTR))
return 0;

folio = virt_to_folio(object);

if (unlikely(!folio_test_slab(folio)))
return folio_size(folio);

return slab_ksize(folio_slab(folio)->slab_cache);
}
EXPORT_SYMBOL(__ksize);

void kfree(const void *x)
{
struct folio *folio;
struct slab *slab;
void *object = (void *)x;

trace_kfree(_RET_IP_, x);

if (unlikely(ZERO_OR_NULL_PTR(x)))
return;

folio = virt_to_folio(x);
if (unlikely(!folio_test_slab(folio))) {
free_large_kmalloc(folio, object);
return;
}
slab = folio_slab(folio);
slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_);
}
EXPORT_SYMBOL(kfree);

#define SHRINK_PROMOTE_MAX 32

/*
Expand Down Expand Up @@ -4863,13 +4783,6 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
return 0;
}

void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
int node, unsigned long caller)
{
return __do_kmalloc_node(size, gfpflags, node, caller);
}
EXPORT_SYMBOL(__kmalloc_node_track_caller);

#ifdef CONFIG_SYSFS
static int count_inuse(struct slab *slab)
{
Expand Down

0 comments on commit b140513

Please sign in to comment.