Skip to content

Commit

Permalink
memcontrol: add helpers for hugetlb memcg accounting
Browse files Browse the repository at this point in the history 
Patch series "hugetlb memcg accounting", v4.

Currently, hugetlb memory usage is not acounted for in the memory
controller, which could lead to memory overprotection for cgroups with
hugetlb-backed memory.  This has been observed in our production system.

For instance, here is one of our usecases: suppose there are two 32G
containers.  The machine is booted with hugetlb_cma=6G, and each container
may or may not use up to 3 gigantic page, depending on the workload within
it.  The rest is anon, cache, slab, etc.  We can set the hugetlb cgroup
limit of each cgroup to 3G to enforce hugetlb fairness.  But it is very
difficult to configure memory.max to keep overall consumption, including
anon, cache, slab etcetera fair.

What we have had to resort to is to constantly poll hugetlb usage and
readjust memory.max.  Similar procedure is done to other memory limits
(memory.low for e.g).  However, this is rather cumbersome and buggy. 
Furthermore, when there is a delay in memory limits correction, (for e.g
when hugetlb usage changes within consecutive runs of the userspace
agent), the system could be in an over/underprotected state.

This patch series rectifies this issue by charging the memcg when the
hugetlb folio is allocated, and uncharging when the folio is freed.  In
addition, a new selftest is added to demonstrate and verify this new
behavior.


This patch (of 4):

This patch exposes charge committing and cancelling as parts of the memory
controller interface.  These functionalities are useful when the
try_charge() and commit_charge() stages have to be separated by other
actions in between (which can fail).  One such example is the new hugetlb
accounting behavior in the following patch.

The patch also adds a helper function to obtain a reference to the
current task's memcg.

Link: https://lkml.kernel.org/r/20231006184629.155543-1-nphamcs@gmail.com
Link: https://lkml.kernel.org/r/20231006184629.155543-2-nphamcs@gmail.com
Signed-off-by: Nhat Pham <nphamcs@gmail.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Frank van der Linden <fvdl@google.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Rik van Riel <riel@surriel.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Tejun heo <tj@kernel.org>
Cc: Yosry Ahmed <yosryahmed@google.com>
Cc: Zefan Li <lizefan.x@bytedance.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  • Loading branch information
@nhatsmrt @akpm00
nhatsmrt authored and akpm00 committed Oct 18, 2023
1 parent 59838b2 commit 4b56938
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 12 deletions.
21 changes: 21 additions & 0 deletions include/linux/memcontrol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -652,6 +652,8 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
page_counter_read(&memcg->memory);
}

void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg);

int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);

/**
Expand Down Expand Up @@ -703,6 +705,8 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
__mem_cgroup_uncharge_list(page_list);
}

void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages);

void mem_cgroup_migrate(struct folio *old, struct folio *new);

/**
Expand Down Expand Up @@ -759,6 +763,8 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);

struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);

struct mem_cgroup *get_mem_cgroup_from_current(void);

struct lruvec *folio_lruvec_lock(struct folio *folio);
struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
Expand Down Expand Up @@ -1239,6 +1245,11 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
return false;
}

static inline void mem_cgroup_commit_charge(struct folio *folio,
struct mem_cgroup *memcg)
{
}

static inline int mem_cgroup_charge(struct folio *folio,
struct mm_struct *mm, gfp_t gfp)
{
Expand All @@ -1263,6 +1274,11 @@ static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
{
}

static inline void mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
unsigned int nr_pages)
{
}

static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)
{
}
Expand Down Expand Up @@ -1300,6 +1316,11 @@ static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
return NULL;
}

static inline struct mem_cgroup *get_mem_cgroup_from_current(void)
{
return NULL;
}

static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css)
{
Expand Down
59 changes: 47 additions & 12 deletions mm/memcontrol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,27 @@ static __always_inline bool memcg_kmem_bypass(void)
return false;
}

/**
* get_mem_cgroup_from_current - Obtain a reference on current task's memcg.
*/
struct mem_cgroup *get_mem_cgroup_from_current(void)
{
struct mem_cgroup *memcg;

if (mem_cgroup_disabled())
return NULL;

again:
rcu_read_lock();
memcg = mem_cgroup_from_task(current);
if (!css_tryget(&memcg->css)) {
rcu_read_unlock();
goto again;
}
rcu_read_unlock();
return memcg;
}

/**
* mem_cgroup_iter - iterate over memory cgroup hierarchy
* @root: hierarchy root
Expand Down Expand Up @@ -2873,7 +2894,12 @@ static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
return try_charge_memcg(memcg, gfp_mask, nr_pages);
}

static inline void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
/**
* mem_cgroup_cancel_charge() - cancel an uncommitted try_charge() call.
* @memcg: memcg previously charged.
* @nr_pages: number of pages previously charged.
*/
void mem_cgroup_cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
{
if (mem_cgroup_is_root(memcg))
return;
Expand All @@ -2898,6 +2924,22 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg)
folio->memcg_data = (unsigned long)memcg;
}

/**
* mem_cgroup_commit_charge - commit a previously successful try_charge().
* @folio: folio to commit the charge to.
* @memcg: memcg previously charged.
*/
void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
{
css_get(&memcg->css);
commit_charge(folio, memcg);

local_irq_disable();
mem_cgroup_charge_statistics(memcg, folio_nr_pages(folio));
memcg_check_events(memcg, folio_nid(folio));
local_irq_enable();
}

#ifdef CONFIG_MEMCG_KMEM
/*
* The allocated objcg pointers array is not accounted directly.
Expand Down Expand Up @@ -6116,15 +6158,15 @@ static void __mem_cgroup_clear_mc(void)

/* we must uncharge all the leftover precharges from mc.to */
if (mc.precharge) {
cancel_charge(mc.to, mc.precharge);
mem_cgroup_cancel_charge(mc.to, mc.precharge);
mc.precharge = 0;
}
/*
* we didn't uncharge from mc.from at mem_cgroup_move_account(), so
* we must uncharge here.
*/
if (mc.moved_charge) {
cancel_charge(mc.from, mc.moved_charge);
mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
mc.moved_charge = 0;
}
/* we must fixup refcnts and charges */
Expand Down Expand Up @@ -7031,20 +7073,13 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
gfp_t gfp)
{
long nr_pages = folio_nr_pages(folio);
int ret;

ret = try_charge(memcg, gfp, nr_pages);
ret = try_charge(memcg, gfp, folio_nr_pages(folio));
if (ret)
goto out;

css_get(&memcg->css);
commit_charge(folio, memcg);

local_irq_disable();
mem_cgroup_charge_statistics(memcg, nr_pages);
memcg_check_events(memcg, folio_nid(folio));
local_irq_enable();
mem_cgroup_commit_charge(folio, memcg);
out:
return ret;
}
Expand Down

0 comments on commit 4b56938

Please sign in to comment.