Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bpf: Make update_prog_stats() always_inline 

The function update_prog_stats() will be called in the bpf trampoline.
In most cases, it will be optimized by the compiler by making it inline.
However, we can't rely on the compiler all the time, and just make it
__always_inline to reduce the possible overhead.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Link: https://lore.kernel.org/r/20250621045501.101187-1-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Menglong Dong 2025-06-21 12:55:01 +08:00 committed by Alexei Starovoitov
parent 99fe8af069
commit c11f34e300
1 changed files with 22 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
kernel/bpf/trampoline.c
View File

@ -911,27 +911,32 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
return bpf_prog_start_time();
}
static void notrace update_prog_stats(struct bpf_prog *prog,
u64 start)
static void notrace __update_prog_stats(struct bpf_prog *prog, u64 start)
{
struct bpf_prog_stats *stats;
unsigned long flags;
u64 duration;
if (static_branch_unlikely(&bpf_stats_enabled_key) &&
/* static_key could be enabled in __bpf_prog_enter*
* and disabled in __bpf_prog_exit*.
* And vice versa.
* Hence check that 'start' is valid.
*/
start > NO_START_TIME) {
u64 duration = sched_clock() - start;
unsigned long flags;
/*
* static_key could be enabled in __bpf_prog_enter* and disabled in
* __bpf_prog_exit*. And vice versa. Check that 'start' is valid.
*/
if (start <= NO_START_TIME)
return;
stats = this_cpu_ptr(prog->stats);
flags = u64_stats_update_begin_irqsave(&stats->syncp);
u64_stats_inc(&stats->cnt);
u64_stats_add(&stats->nsecs, duration);
u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
duration = sched_clock() - start;
stats = this_cpu_ptr(prog->stats);
flags = u64_stats_update_begin_irqsave(&stats->syncp);
u64_stats_inc(&stats->cnt);
u64_stats_add(&stats->nsecs, duration);
u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
static __always_inline void notrace update_prog_stats(struct bpf_prog *prog,
u64 start)
{
if (static_branch_unlikely(&bpf_stats_enabled_key))
__update_prog_stats(prog, start);
}
static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,