Skip to content

Commit

Permalink
bpf: Remove hard-coded btf_vmlinux assumption from BPF verifier
Browse files Browse the repository at this point in the history 
Remove a permeating assumption thoughout BPF verifier of vmlinux BTF. Instead,
wherever BTF type IDs are involved, also track the instance of struct btf that
goes along with the type ID. This allows to gradually add support for kernel
module BTFs and using/tracking module types across BPF helper calls and
registers.

This patch also renames btf_id() function to btf_obj_id() to minimize naming
clash with using btf_id to denote BTF *type* ID, rather than BTF *object*'s ID.

Also, altough btf_vmlinux can't get destructed and thus doesn't need
refcounting, module BTFs need that, so apply BTF refcounting universally when
BPF program is using BTF-powered attachment (tp_btf, fentry/fexit, etc). This
makes for simpler clean up code.

Now that BTF type ID is not enough to uniquely identify a BTF type, extend BPF
trampoline key to include BTF object ID. To differentiate that from target
program BPF ID, set 31st bit of type ID. BTF type IDs (at least currently) are
not allowed to take full 32 bits, so there is no danger of confusing that bit
with a valid BTF type ID.

Signed-off-by: Andrii Nakryiko <[email protected]>
Signed-off-by: Alexei Starovoitov <[email protected]>
Link: https://lore.kernel.org/bpf/[email protected]
  • Loading branch information
@anakryiko
anakryiko authored and Alexei Starovoitov committed Dec 4, 2020
1 parent 6bcd39d commit 22dc4a0
Show file tree
Hide file tree
Showing 7 changed files with 148 additions and 67 deletions.
13 changes: 9 additions & 4 deletions include/linux/bpf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,10 @@ struct bpf_insn_access_aux {
enum bpf_reg_type reg_type;
union {
int ctx_field_size;
u32 btf_id;
struct {
struct btf *btf;
u32 btf_id;
};
};
struct bpf_verifier_log *log; /* for verbose logs */
};
Expand Down Expand Up @@ -458,6 +461,7 @@ struct bpf_verifier_ops {
struct bpf_insn *dst,
struct bpf_prog *prog, u32 *target_size);
int (*btf_struct_access)(struct bpf_verifier_log *log,
const struct btf *btf,
const struct btf_type *t, int off, int size,
enum bpf_access_type atype,
u32 *next_btf_id);
Expand Down Expand Up @@ -771,6 +775,7 @@ struct bpf_prog_aux {
u32 ctx_arg_info_size;
u32 max_rdonly_access;
u32 max_rdwr_access;
struct btf *attach_btf;
const struct bpf_ctx_arg_aux *ctx_arg_info;
struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
struct bpf_prog *dst_prog;
Expand Down Expand Up @@ -1005,7 +1010,6 @@ struct bpf_event_entry {

bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
int bpf_prog_calc_tag(struct bpf_prog *fp);
const char *kernel_type_name(u32 btf_type_id);

const struct bpf_func_proto *bpf_get_trace_printk_proto(void);

Expand Down Expand Up @@ -1450,12 +1454,13 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
const struct bpf_prog *prog,
struct bpf_insn_access_aux *info);
int btf_struct_access(struct bpf_verifier_log *log,
int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, int off, int size,
enum bpf_access_type atype,
u32 *next_btf_id);
bool btf_struct_ids_match(struct bpf_verifier_log *log,
int off, u32 id, u32 need_type_id);
const struct btf *btf, u32 id, int off,
const struct btf *need_btf, u32 need_type_id);

int btf_distill_func_proto(struct bpf_verifier_log *log,
struct btf *btf,
Expand Down
28 changes: 21 additions & 7 deletions include/linux/bpf_verifier.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#define _LINUX_BPF_VERIFIER_H 1

#include <linux/bpf.h> /* for enum bpf_reg_type */
#include <linux/btf.h> /* for struct btf and btf_id() */
#include <linux/filter.h> /* for MAX_BPF_STACK */
#include <linux/tnum.h>

Expand Down Expand Up @@ -43,6 +44,8 @@ enum bpf_reg_liveness {
struct bpf_reg_state {
/* Ordering of fields matters. See states_equal() */
enum bpf_reg_type type;
/* Fixed part of pointer offset, pointer types only */
s32 off;
union {
/* valid when type == PTR_TO_PACKET */
int range;
Expand All @@ -52,15 +55,20 @@ struct bpf_reg_state {
*/
struct bpf_map *map_ptr;

u32 btf_id; /* for PTR_TO_BTF_ID */
/* for PTR_TO_BTF_ID */
struct {
struct btf *btf;
u32 btf_id;
};

u32 mem_size; /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */

/* Max size from any of the above. */
unsigned long raw;
struct {
unsigned long raw1;
unsigned long raw2;
} raw;
};
/* Fixed part of pointer offset, pointer types only */
s32 off;
/* For PTR_TO_PACKET, used to find other pointers with the same variable
* offset, so they can share range knowledge.
* For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we
Expand Down Expand Up @@ -311,7 +319,10 @@ struct bpf_insn_aux_data {
struct {
enum bpf_reg_type reg_type; /* type of pseudo_btf_id */
union {
u32 btf_id; /* btf_id for struct typed var */
struct {
struct btf *btf;
u32 btf_id; /* btf_id for struct typed var */
};
u32 mem_size; /* mem_size for non-struct typed var */
};
} btf_var;
Expand Down Expand Up @@ -459,9 +470,12 @@ int check_ctx_reg(struct bpf_verifier_env *env,

/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
u32 btf_id)
struct btf *btf, u32 btf_id)
{
return tgt_prog ? (((u64)tgt_prog->aux->id) << 32 | btf_id) : btf_id;
if (tgt_prog)
return ((u64)tgt_prog->aux->id << 32) | btf_id;
else
return ((u64)btf_obj_id(btf) << 32) | 0x80000000 | btf_id;
}

int bpf_check_attach_target(struct bpf_verifier_log *log,
Expand Down
5 changes: 4 additions & 1 deletion include/linux/btf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ struct btf_show;

extern const struct file_operations btf_fops;

void btf_get(struct btf *btf);
void btf_put(struct btf *btf);
int btf_new_fd(const union bpf_attr *attr);
struct btf *btf_get_by_fd(int fd);
Expand Down Expand Up @@ -88,7 +89,7 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj,
char *buf, int len, u64 flags);

int btf_get_fd_by_id(u32 id);
u32 btf_id(const struct btf *btf);
u32 btf_obj_id(const struct btf *btf);
bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
const struct btf_member *m,
u32 expected_offset, u32 expected_size);
Expand Down Expand Up @@ -206,6 +207,8 @@ static inline const struct btf_var_secinfo *btf_type_var_secinfo(
}

#ifdef CONFIG_BPF_SYSCALL
struct bpf_prog;

const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
const char *btf_name_by_offset(const struct btf *btf, u32 offset);
struct btf *btf_parse_vmlinux(void);
Expand Down
65 changes: 44 additions & 21 deletions kernel/bpf/btf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1524,6 +1524,11 @@ static void btf_free_rcu(struct rcu_head *rcu)
btf_free(btf);
}

void btf_get(struct btf *btf)
{
refcount_inc(&btf->refcnt);
}

void btf_put(struct btf *btf)
{
if (btf && refcount_dec_and_test(&btf->refcnt)) {
Expand Down Expand Up @@ -4555,11 +4560,10 @@ struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)
{
struct bpf_prog *tgt_prog = prog->aux->dst_prog;

if (tgt_prog) {
if (tgt_prog)
return tgt_prog->aux->btf;
} else {
return btf_vmlinux;
}
else
return prog->aux->attach_btf;
}

static bool is_string_ptr(struct btf *btf, const struct btf_type *t)
Expand Down Expand Up @@ -4700,6 +4704,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,

if (ctx_arg_info->offset == off) {
info->reg_type = ctx_arg_info->reg_type;
info->btf = btf_vmlinux;
info->btf_id = ctx_arg_info->btf_id;
return true;
}
Expand All @@ -4716,13 +4721,15 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,

ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg);
if (ret > 0) {
info->btf = btf_vmlinux;
info->btf_id = ret;
return true;
} else {
return false;
}
}

info->btf = btf;
info->btf_id = t->type;
t = btf_type_by_id(btf, t->type);
/* skip modifiers */
Expand All @@ -4749,7 +4756,7 @@ enum bpf_struct_walk_result {
WALK_STRUCT,
};

static int btf_struct_walk(struct bpf_verifier_log *log,
static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, int off, int size,
u32 *next_btf_id)
{
Expand All @@ -4760,7 +4767,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log,
u32 vlen, elem_id, mid;

again:
tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
tname = __btf_name_by_offset(btf, t->name_off);
if (!btf_type_is_struct(t)) {
bpf_log(log, "Type '%s' is not a struct\n", tname);
return -EINVAL;
Expand All @@ -4777,7 +4784,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log,
goto error;

member = btf_type_member(t) + vlen - 1;
mtype = btf_type_skip_modifiers(btf_vmlinux, member->type,
mtype = btf_type_skip_modifiers(btf, member->type,
NULL);
if (!btf_type_is_array(mtype))
goto error;
Expand All @@ -4793,7 +4800,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log,
/* Only allow structure for now, can be relaxed for
* other types later.
*/
t = btf_type_skip_modifiers(btf_vmlinux, array_elem->type,
t = btf_type_skip_modifiers(btf, array_elem->type,
NULL);
if (!btf_type_is_struct(t))
goto error;
Expand Down Expand Up @@ -4851,10 +4858,10 @@ static int btf_struct_walk(struct bpf_verifier_log *log,

/* type of the field */
mid = member->type;
mtype = btf_type_by_id(btf_vmlinux, member->type);
mname = __btf_name_by_offset(btf_vmlinux, member->name_off);
mtype = btf_type_by_id(btf, member->type);
mname = __btf_name_by_offset(btf, member->name_off);

mtype = __btf_resolve_size(btf_vmlinux, mtype, &msize,
mtype = __btf_resolve_size(btf, mtype, &msize,
&elem_type, &elem_id, &total_nelems,
&mid);
if (IS_ERR(mtype)) {
Expand Down Expand Up @@ -4949,7 +4956,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log,
mname, moff, tname, off, size);
return -EACCES;
}
stype = btf_type_skip_modifiers(btf_vmlinux, mtype->type, &id);
stype = btf_type_skip_modifiers(btf, mtype->type, &id);
if (btf_type_is_struct(stype)) {
*next_btf_id = id;
return WALK_PTR;
Expand All @@ -4975,7 +4982,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log,
return -EINVAL;
}

int btf_struct_access(struct bpf_verifier_log *log,
int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, int off, int size,
enum bpf_access_type atype __maybe_unused,
u32 *next_btf_id)
Expand All @@ -4984,7 +4991,7 @@ int btf_struct_access(struct bpf_verifier_log *log,
u32 id;

do {
err = btf_struct_walk(log, t, off, size, &id);
err = btf_struct_walk(log, btf, t, off, size, &id);

switch (err) {
case WALK_PTR:
Expand All @@ -5000,7 +5007,7 @@ int btf_struct_access(struct bpf_verifier_log *log,
* by diving in it. At this point the offset is
* aligned with the new type, so set it to 0.
*/
t = btf_type_by_id(btf_vmlinux, id);
t = btf_type_by_id(btf, id);
off = 0;
break;
default:
Expand All @@ -5016,21 +5023,37 @@ int btf_struct_access(struct bpf_verifier_log *log,
return -EINVAL;
}

/* Check that two BTF types, each specified as an BTF object + id, are exactly
* the same. Trivial ID check is not enough due to module BTFs, because we can
* end up with two different module BTFs, but IDs point to the common type in
* vmlinux BTF.
*/
static bool btf_types_are_same(const struct btf *btf1, u32 id1,
const struct btf *btf2, u32 id2)
{
if (id1 != id2)
return false;
if (btf1 == btf2)
return true;
return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2);
}

bool btf_struct_ids_match(struct bpf_verifier_log *log,
int off, u32 id, u32 need_type_id)
const struct btf *btf, u32 id, int off,
const struct btf *need_btf, u32 need_type_id)
{
const struct btf_type *type;
int err;

/* Are we already done? */
if (need_type_id == id && off == 0)
if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id))
return true;

again:
type = btf_type_by_id(btf_vmlinux, id);
type = btf_type_by_id(btf, id);
if (!type)
return false;
err = btf_struct_walk(log, type, off, 1, &id);
err = btf_struct_walk(log, btf, type, off, 1, &id);
if (err != WALK_STRUCT)
return false;

Expand All @@ -5039,7 +5062,7 @@ bool btf_struct_ids_match(struct bpf_verifier_log *log,
* continue the search with offset 0 in the new
* type.
*/
if (need_type_id != id) {
if (!btf_types_are_same(btf, id, need_btf, need_type_id)) {
off = 0;
goto again;
}
Expand Down Expand Up @@ -5710,7 +5733,7 @@ int btf_get_fd_by_id(u32 id)
return fd;
}

u32 btf_id(const struct btf *btf)
u32 btf_obj_id(const struct btf *btf)
{
return btf->id;
}
Expand Down
Loading

0 comments on commit 22dc4a0

Please sign in to comment.