Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bpf: verifier: refactor kfunc specialization 

Move kfunc specialization (function address substitution) to later stage
of verification to support a new use case, where we need to take into
consideration whether kfunc is called in sleepable context.

Minor refactoring in add_kfunc_call(), making sure that if function
fails, kfunc desc is not added to tab->descs (previously it could be
added or not, depending on what failed).

Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251026203853.135105-9-mykyta.yatsenko5@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Mykyta Yatsenko 2025-10-26 20:38:51 +00:00 committed by Alexei Starovoitov
parent e3e36edb1b
commit d869d56ca8
1 changed files with 53 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
kernel/bpf/verifier.c
View File

@ -209,8 +209,6 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
static int ref_set_non_owning(struct bpf_verifier_env *env, static int ref_set_non_owning(struct bpf_verifier_env *env,
struct bpf_reg_state *reg); struct bpf_reg_state *reg);
static void specialize_kfunc(struct bpf_verifier_env *env,
u32 func_id, u16 offset, unsigned long *addr);
static bool is_trusted_reg(const struct bpf_reg_state *reg); static bool is_trusted_reg(const struct bpf_reg_state *reg);
static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
@ -3126,6 +3124,8 @@ struct bpf_kfunc_btf_tab {
u32 nr_descs; u32 nr_descs;
}; };
static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc);
static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
{ {
const struct bpf_kfunc_desc *d0 = a; const struct bpf_kfunc_desc *d0 = a;
@ -3143,7 +3143,7 @@ static int kfunc_btf_cmp_by_off(const void *a, const void *b)
return d0->offset - d1->offset; return d0->offset - d1->offset;
} }
static const struct bpf_kfunc_desc * static struct bpf_kfunc_desc *
find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
{ {
struct bpf_kfunc_desc desc = { struct bpf_kfunc_desc desc = {
@ -3266,12 +3266,12 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
{ {
const struct btf_type *func, *func_proto; const struct btf_type *func, *func_proto;
struct bpf_kfunc_btf_tab *btf_tab; struct bpf_kfunc_btf_tab *btf_tab;
struct btf_func_model func_model;
struct bpf_kfunc_desc_tab *tab; struct bpf_kfunc_desc_tab *tab;
struct bpf_prog_aux *prog_aux; struct bpf_prog_aux *prog_aux;
struct bpf_kfunc_desc *desc; struct bpf_kfunc_desc *desc;
const char *func_name; const char *func_name;
struct btf *desc_btf; struct btf *desc_btf;
unsigned long call_imm;
unsigned long addr; unsigned long addr;
int err; int err;
@ -3355,19 +3355,6 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
func_name); func_name);
return -EINVAL; return -EINVAL;
} }
specialize_kfunc(env, func_id, offset, &addr);
if (bpf_jit_supports_far_kfunc_call()) {
call_imm = func_id;
} else {
call_imm = BPF_CALL_IMM(addr);
/* Check whether the relative offset overflows desc->imm */
if ((unsigned long)(s32)call_imm != call_imm) {
verbose(env, "address of kernel function %s is out of range\n",
func_name);
return -EINVAL;
}
}
if (bpf_dev_bound_kfunc_id(func_id)) { if (bpf_dev_bound_kfunc_id(func_id)) {
err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
@ -3375,18 +3362,20 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
return err; return err;
} }
desc = &tab->descs[tab->nr_descs++];
desc->func_id = func_id;
desc->imm = call_imm;
desc->offset = offset;
desc->addr = addr;
err = btf_distill_func_proto(&env->log, desc_btf, err = btf_distill_func_proto(&env->log, desc_btf,
func_proto, func_name, func_proto, func_name,
&desc->func_model); &func_model);
if (!err) if (err)
sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), return err;
kfunc_desc_cmp_by_id_off, NULL);
return err; desc = &tab->descs[tab->nr_descs++];
desc->func_id = func_id;
desc->offset = offset;
desc->addr = addr;
desc->func_model = func_model;
sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
kfunc_desc_cmp_by_id_off, NULL);
return 0;
} }
static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b)
@ -21880,46 +21869,57 @@ static int fixup_call_args(struct bpf_verifier_env *env)
} }
/* replace a generic kfunc with a specialized version if necessary */ /* replace a generic kfunc with a specialized version if necessary */
static void specialize_kfunc(struct bpf_verifier_env *env, static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc)
u32 func_id, u16 offset, unsigned long *addr)
{ {
struct bpf_prog *prog = env->prog; struct bpf_prog *prog = env->prog;
bool seen_direct_write; bool seen_direct_write;
void *xdp_kfunc; void *xdp_kfunc;
bool is_rdonly; bool is_rdonly;
u32 func_id = desc->func_id;
u16 offset = desc->offset;
unsigned long addr = desc->addr, call_imm;
if (offset) /* return if module BTF is used */
goto set_imm;
if (bpf_dev_bound_kfunc_id(func_id)) { if (bpf_dev_bound_kfunc_id(func_id)) {
xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id); xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);
if (xdp_kfunc) { if (xdp_kfunc)
*addr = (unsigned long)xdp_kfunc; addr = (unsigned long)xdp_kfunc;
return;
}
/* fallback to default kfunc when not supported by netdev */ /* fallback to default kfunc when not supported by netdev */
} } else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
if (offset)
return;
if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
seen_direct_write = env->seen_direct_write; seen_direct_write = env->seen_direct_write;
is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
if (is_rdonly) if (is_rdonly)
*addr = (unsigned long)bpf_dynptr_from_skb_rdonly; addr = (unsigned long)bpf_dynptr_from_skb_rdonly;
/* restore env->seen_direct_write to its original value, since /* restore env->seen_direct_write to its original value, since
* may_access_direct_pkt_data mutates it * may_access_direct_pkt_data mutates it
*/ */
env->seen_direct_write = seen_direct_write; env->seen_direct_write = seen_direct_write;
} else if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr]) {
if (bpf_lsm_has_d_inode_locked(prog))
addr = (unsigned long)bpf_set_dentry_xattr_locked;
} else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) {
if (bpf_lsm_has_d_inode_locked(prog))
addr = (unsigned long)bpf_remove_dentry_xattr_locked;
} }
if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr] && set_imm:
bpf_lsm_has_d_inode_locked(prog)) if (bpf_jit_supports_far_kfunc_call()) {
*addr = (unsigned long)bpf_set_dentry_xattr_locked; call_imm = func_id;
} else {
if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr] && call_imm = BPF_CALL_IMM(addr);
bpf_lsm_has_d_inode_locked(prog)) /* Check whether the relative offset overflows desc->imm */
*addr = (unsigned long)bpf_remove_dentry_xattr_locked; if ((unsigned long)(s32)call_imm != call_imm) {
verbose(env, "address of kernel func_id %u is out of range\n", func_id);
return -EINVAL;
}
}
desc->imm = call_imm;
desc->addr = addr;
return 0;
} }
static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
@ -21942,7 +21942,8 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_insn *insn_buf, int insn_idx, int *cnt) struct bpf_insn *insn_buf, int insn_idx, int *cnt)
{ {
const struct bpf_kfunc_desc *desc; struct bpf_kfunc_desc *desc;
int err;
if (!insn->imm) { if (!insn->imm) {
verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
@ -21962,6 +21963,10 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
return -EFAULT; return -EFAULT;
} }
err = specialize_kfunc(env, desc);
if (err)
return err;
if (!bpf_jit_supports_far_kfunc_call()) if (!bpf_jit_supports_far_kfunc_call())
insn->imm = BPF_CALL_IMM(desc->addr); insn->imm = BPF_CALL_IMM(desc->addr);
if (insn->off) if (insn->off)