Skip to content

Commit

Permalink
jump_label: Do not profile branch annotations
Browse files Browse the repository at this point in the history
While running my branch profiler that checks for incorrect "likely" and
"unlikely"s around the kernel, there's a large number of them that are
incorrect due to being "static_branches".

As static_branches are rather special, as they are likely or unlikely for
other reasons than normal annotations are used for, there's no reason to
have them be profiled.

Expose the "unlikely_notrace" and "likely_notrace" so that the
static_branch can use them, and have them be ignored by the branch
profilers.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20201211163754.585174b9@gandalf.local.home
  • Loading branch information
rostedt authored and Peter Zijlstra committed Jan 22, 2021
1 parent c753925 commit 2f0df49
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 6 deletions.
2 changes: 2 additions & 0 deletions include/linux/compiler.h
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
#else
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
# define likely_notrace(x) likely(x)
# define unlikely_notrace(x) unlikely(x)
#endif

/* Optimization barrier */
Expand Down
12 changes: 6 additions & 6 deletions include/linux/jump_label.h
Original file line number Diff line number Diff line change
Expand Up @@ -261,14 +261,14 @@ static __always_inline void jump_label_init(void)

static __always_inline bool static_key_false(struct static_key *key)
{
if (unlikely(static_key_count(key) > 0))
if (unlikely_notrace(static_key_count(key) > 0))
return true;
return false;
}

static __always_inline bool static_key_true(struct static_key *key)
{
if (likely(static_key_count(key) > 0))
if (likely_notrace(static_key_count(key) > 0))
return true;
return false;
}
Expand Down Expand Up @@ -460,7 +460,7 @@ extern bool ____wrong_branch_error(void);
branch = !arch_static_branch_jump(&(x)->key, true); \
else \
branch = ____wrong_branch_error(); \
likely(branch); \
likely_notrace(branch); \
})

#define static_branch_unlikely(x) \
Expand All @@ -472,13 +472,13 @@ extern bool ____wrong_branch_error(void);
branch = arch_static_branch(&(x)->key, false); \
else \
branch = ____wrong_branch_error(); \
unlikely(branch); \
unlikely_notrace(branch); \
})

#else /* !CONFIG_JUMP_LABEL */

#define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
#define static_branch_likely(x) likely_notrace(static_key_enabled(&(x)->key))
#define static_branch_unlikely(x) unlikely_notrace(static_key_enabled(&(x)->key))

#endif /* CONFIG_JUMP_LABEL */

Expand Down

0 comments on commit 2f0df49

Please sign in to comment.