Skip to content

Commit 27ae799

Browse files
iamkafaiAlexei Starovoitov
iamkafai
authored and
Alexei Starovoitov
committed
bpf: Introduce BPF_PROG_TYPE_STRUCT_OPS
This patch allows the kernel's struct ops (i.e. func ptr) to be implemented in BPF. The first use case in this series is the "struct tcp_congestion_ops" which will be introduced in a latter patch. This patch introduces a new prog type BPF_PROG_TYPE_STRUCT_OPS. The BPF_PROG_TYPE_STRUCT_OPS prog is verified against a particular func ptr of a kernel struct. The attr->attach_btf_id is the btf id of a kernel struct. The attr->expected_attach_type is the member "index" of that kernel struct. The first member of a struct starts with member index 0. That will avoid ambiguity when a kernel struct has multiple func ptrs with the same func signature. For example, a BPF_PROG_TYPE_STRUCT_OPS prog is written to implement the "init" func ptr of the "struct tcp_congestion_ops". The attr->attach_btf_id is the btf id of the "struct tcp_congestion_ops" of the _running_ kernel. The attr->expected_attach_type is 3. The ctx of BPF_PROG_TYPE_STRUCT_OPS is an array of u64 args saved by arch_prepare_bpf_trampoline that will be done in the next patch when introducing BPF_MAP_TYPE_STRUCT_OPS. "struct bpf_struct_ops" is introduced as a common interface for the kernel struct that supports BPF_PROG_TYPE_STRUCT_OPS prog. The supporting kernel struct will need to implement an instance of the "struct bpf_struct_ops". The supporting kernel struct also needs to implement a bpf_verifier_ops. During BPF_PROG_LOAD, bpf_struct_ops_find() will find the right bpf_verifier_ops by searching the attr->attach_btf_id. A new "btf_struct_access" is also added to the bpf_verifier_ops such that the supporting kernel struct can optionally provide its own specific check on accessing the func arg (e.g. provide limited write access). After btf_vmlinux is parsed, the new bpf_struct_ops_init() is called to initialize some values (e.g. the btf id of the supporting kernel struct) and it can only be done once the btf_vmlinux is available. The R0 checks at BPF_EXIT is excluded for the BPF_PROG_TYPE_STRUCT_OPS prog if the return type of the prog->aux->attach_func_proto is "void". Signed-off-by: Martin KaFai Lau <[email protected]> Signed-off-by: Alexei Starovoitov <[email protected]> Acked-by: Andrii Nakryiko <[email protected]> Acked-by: Yonghong Song <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent 976aba0 commit 27ae799

File tree

10 files changed

+373
-63
lines changed

10 files changed

+373
-63
lines changed

‎include/linux/bpf.h

+30
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,10 @@ struct bpf_verifier_ops {
349349
const struct bpf_insn *src,
350350
struct bpf_insn *dst,
351351
struct bpf_prog *prog, u32 *target_size);
352+
int (*btf_struct_access)(struct bpf_verifier_log *log,
353+
const struct btf_type *t, int off, int size,
354+
enum bpf_access_type atype,
355+
u32 *next_btf_id);
352356
};
353357

354358
struct bpf_prog_offload_ops {
@@ -668,6 +672,32 @@ struct bpf_array_aux {
668672
struct work_struct work;
669673
};
670674

675+
struct btf_type;
676+
struct btf_member;
677+
678+
#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
679+
struct bpf_struct_ops {
680+
const struct bpf_verifier_ops *verifier_ops;
681+
int (*init)(struct btf *btf);
682+
int (*check_member)(const struct btf_type *t,
683+
const struct btf_member *member);
684+
const struct btf_type *type;
685+
const char *name;
686+
struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
687+
u32 type_id;
688+
};
689+
690+
#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
691+
const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
692+
void bpf_struct_ops_init(struct btf *btf);
693+
#else
694+
static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
695+
{
696+
return NULL;
697+
}
698+
static inline void bpf_struct_ops_init(struct btf *btf) { }
699+
#endif
700+
671701
struct bpf_array {
672702
struct bpf_map map;
673703
u32 elem_size;

‎include/linux/bpf_types.h

+4
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LIRC_MODE2, lirc_mode2,
6565
BPF_PROG_TYPE(BPF_PROG_TYPE_SK_REUSEPORT, sk_reuseport,
6666
struct sk_reuseport_md, struct sk_reuseport_kern)
6767
#endif
68+
#if defined(CONFIG_BPF_JIT)
69+
BPF_PROG_TYPE(BPF_PROG_TYPE_STRUCT_OPS, bpf_struct_ops,
70+
void *, void *)
71+
#endif
6872

6973
BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
7074
BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops)

‎include/linux/btf.h

+34
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,18 @@ bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
5353
u32 expected_offset, u32 expected_size);
5454
int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t);
5555
bool btf_type_is_void(const struct btf_type *t);
56+
s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind);
57+
const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
58+
u32 id, u32 *res_id);
59+
const struct btf_type *btf_type_resolve_ptr(const struct btf *btf,
60+
u32 id, u32 *res_id);
61+
const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf,
62+
u32 id, u32 *res_id);
63+
64+
#define for_each_member(i, struct_type, member) \
65+
for (i = 0, member = btf_type_member(struct_type); \
66+
i < btf_type_vlen(struct_type); \
67+
i++, member++)
5668

5769
static inline bool btf_type_is_ptr(const struct btf_type *t)
5870
{
@@ -84,6 +96,28 @@ static inline bool btf_type_is_func_proto(const struct btf_type *t)
8496
return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
8597
}
8698

99+
static inline u16 btf_type_vlen(const struct btf_type *t)
100+
{
101+
return BTF_INFO_VLEN(t->info);
102+
}
103+
104+
static inline bool btf_type_kflag(const struct btf_type *t)
105+
{
106+
return BTF_INFO_KFLAG(t->info);
107+
}
108+
109+
static inline u32 btf_member_bitfield_size(const struct btf_type *struct_type,
110+
const struct btf_member *member)
111+
{
112+
return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
113+
: 0;
114+
}
115+
116+
static inline const struct btf_member *btf_type_member(const struct btf_type *t)
117+
{
118+
return (const struct btf_member *)(t + 1);
119+
}
120+
87121
#ifdef CONFIG_BPF_SYSCALL
88122
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
89123
const char *btf_name_by_offset(const struct btf *btf, u32 offset);

‎include/uapi/linux/bpf.h

+1
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ enum bpf_prog_type {
174174
BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
175175
BPF_PROG_TYPE_CGROUP_SOCKOPT,
176176
BPF_PROG_TYPE_TRACING,
177+
BPF_PROG_TYPE_STRUCT_OPS,
177178
};
178179

179180
enum bpf_attach_type {

‎kernel/bpf/Makefile

+3
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,6 @@ endif
2727
ifeq ($(CONFIG_SYSFS),y)
2828
obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o
2929
endif
30+
ifeq ($(CONFIG_BPF_JIT),y)
31+
obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
32+
endif

‎kernel/bpf/bpf_struct_ops.c

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/* Copyright (c) 2019 Facebook */
3+
4+
#include <linux/bpf.h>
5+
#include <linux/bpf_verifier.h>
6+
#include <linux/btf.h>
7+
#include <linux/filter.h>
8+
#include <linux/slab.h>
9+
#include <linux/numa.h>
10+
#include <linux/seq_file.h>
11+
#include <linux/refcount.h>
12+
13+
#define BPF_STRUCT_OPS_TYPE(_name) \
14+
extern struct bpf_struct_ops bpf_##_name;
15+
#include "bpf_struct_ops_types.h"
16+
#undef BPF_STRUCT_OPS_TYPE
17+
18+
enum {
19+
#define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name,
20+
#include "bpf_struct_ops_types.h"
21+
#undef BPF_STRUCT_OPS_TYPE
22+
__NR_BPF_STRUCT_OPS_TYPE,
23+
};
24+
25+
static struct bpf_struct_ops * const bpf_struct_ops[] = {
26+
#define BPF_STRUCT_OPS_TYPE(_name) \
27+
[BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name,
28+
#include "bpf_struct_ops_types.h"
29+
#undef BPF_STRUCT_OPS_TYPE
30+
};
31+
32+
const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
33+
};
34+
35+
const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
36+
};
37+
38+
void bpf_struct_ops_init(struct btf *btf)
39+
{
40+
const struct btf_member *member;
41+
struct bpf_struct_ops *st_ops;
42+
struct bpf_verifier_log log = {};
43+
const struct btf_type *t;
44+
const char *mname;
45+
s32 type_id;
46+
u32 i, j;
47+
48+
for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
49+
st_ops = bpf_struct_ops[i];
50+
51+
type_id = btf_find_by_name_kind(btf, st_ops->name,
52+
BTF_KIND_STRUCT);
53+
if (type_id < 0) {
54+
pr_warn("Cannot find struct %s in btf_vmlinux\n",
55+
st_ops->name);
56+
continue;
57+
}
58+
t = btf_type_by_id(btf, type_id);
59+
if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
60+
pr_warn("Cannot support #%u members in struct %s\n",
61+
btf_type_vlen(t), st_ops->name);
62+
continue;
63+
}
64+
65+
for_each_member(j, t, member) {
66+
const struct btf_type *func_proto;
67+
68+
mname = btf_name_by_offset(btf, member->name_off);
69+
if (!*mname) {
70+
pr_warn("anon member in struct %s is not supported\n",
71+
st_ops->name);
72+
break;
73+
}
74+
75+
if (btf_member_bitfield_size(t, member)) {
76+
pr_warn("bit field member %s in struct %s is not supported\n",
77+
mname, st_ops->name);
78+
break;
79+
}
80+
81+
func_proto = btf_type_resolve_func_ptr(btf,
82+
member->type,
83+
NULL);
84+
if (func_proto &&
85+
btf_distill_func_proto(&log, btf,
86+
func_proto, mname,
87+
&st_ops->func_models[j])) {
88+
pr_warn("Error in parsing func ptr %s in struct %s\n",
89+
mname, st_ops->name);
90+
break;
91+
}
92+
}
93+
94+
if (j == btf_type_vlen(t)) {
95+
if (st_ops->init(btf)) {
96+
pr_warn("Error in init bpf_struct_ops %s\n",
97+
st_ops->name);
98+
} else {
99+
st_ops->type_id = type_id;
100+
st_ops->type = t;
101+
}
102+
}
103+
}
104+
}
105+
106+
extern struct btf *btf_vmlinux;
107+
108+
const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
109+
{
110+
unsigned int i;
111+
112+
if (!type_id || !btf_vmlinux)
113+
return NULL;
114+
115+
for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
116+
if (bpf_struct_ops[i]->type_id == type_id)
117+
return bpf_struct_ops[i];
118+
}
119+
120+
return NULL;
121+
}

‎kernel/bpf/bpf_struct_ops_types.h

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/* internal file - do not include directly */
3+
4+
/* To be filled in a later patch */

‎kernel/bpf/btf.c

+61-27
Original file line numberDiff line numberDiff line change
@@ -180,11 +180,6 @@
180180
*/
181181
#define BTF_MAX_SIZE (16 * 1024 * 1024)
182182

183-
#define for_each_member(i, struct_type, member) \
184-
for (i = 0, member = btf_type_member(struct_type); \
185-
i < btf_type_vlen(struct_type); \
186-
i++, member++)
187-
188183
#define for_each_member_from(i, from, struct_type, member) \
189184
for (i = from, member = btf_type_member(struct_type) + from; \
190185
i < btf_type_vlen(struct_type); \
@@ -382,6 +377,65 @@ static bool btf_type_is_datasec(const struct btf_type *t)
382377
return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
383378
}
384379

380+
s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind)
381+
{
382+
const struct btf_type *t;
383+
const char *tname;
384+
u32 i;
385+
386+
for (i = 1; i <= btf->nr_types; i++) {
387+
t = btf->types[i];
388+
if (BTF_INFO_KIND(t->info) != kind)
389+
continue;
390+
391+
tname = btf_name_by_offset(btf, t->name_off);
392+
if (!strcmp(tname, name))
393+
return i;
394+
}
395+
396+
return -ENOENT;
397+
}
398+
399+
const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
400+
u32 id, u32 *res_id)
401+
{
402+
const struct btf_type *t = btf_type_by_id(btf, id);
403+
404+
while (btf_type_is_modifier(t)) {
405+
id = t->type;
406+
t = btf_type_by_id(btf, t->type);
407+
}
408+
409+
if (res_id)
410+
*res_id = id;
411+
412+
return t;
413+
}
414+
415+
const struct btf_type *btf_type_resolve_ptr(const struct btf *btf,
416+
u32 id, u32 *res_id)
417+
{
418+
const struct btf_type *t;
419+
420+
t = btf_type_skip_modifiers(btf, id, NULL);
421+
if (!btf_type_is_ptr(t))
422+
return NULL;
423+
424+
return btf_type_skip_modifiers(btf, t->type, res_id);
425+
}
426+
427+
const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf,
428+
u32 id, u32 *res_id)
429+
{
430+
const struct btf_type *ptype;
431+
432+
ptype = btf_type_resolve_ptr(btf, id, res_id);
433+
if (ptype && btf_type_is_func_proto(ptype))
434+
return ptype;
435+
436+
return NULL;
437+
}
438+
385439
/* Types that act only as a source, not sink or intermediate
386440
* type when resolving.
387441
*/
@@ -446,30 +500,13 @@ static const char *btf_int_encoding_str(u8 encoding)
446500
return "UNKN";
447501
}
448502

449-
static u16 btf_type_vlen(const struct btf_type *t)
450-
{
451-
return BTF_INFO_VLEN(t->info);
452-
}
453-
454-
static bool btf_type_kflag(const struct btf_type *t)
455-
{
456-
return BTF_INFO_KFLAG(t->info);
457-
}
458-
459503
static u32 btf_member_bit_offset(const struct btf_type *struct_type,
460504
const struct btf_member *member)
461505
{
462506
return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
463507
: member->offset;
464508
}
465509

466-
static u32 btf_member_bitfield_size(const struct btf_type *struct_type,
467-
const struct btf_member *member)
468-
{
469-
return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
470-
: 0;
471-
}
472-
473510
static u32 btf_type_int(const struct btf_type *t)
474511
{
475512
return *(u32 *)(t + 1);
@@ -480,11 +517,6 @@ static const struct btf_array *btf_type_array(const struct btf_type *t)
480517
return (const struct btf_array *)(t + 1);
481518
}
482519

483-
static const struct btf_member *btf_type_member(const struct btf_type *t)
484-
{
485-
return (const struct btf_member *)(t + 1);
486-
}
487-
488520
static const struct btf_enum *btf_type_enum(const struct btf_type *t)
489521
{
490522
return (const struct btf_enum *)(t + 1);
@@ -3605,6 +3637,8 @@ struct btf *btf_parse_vmlinux(void)
36053637
goto errout;
36063638
}
36073639

3640+
bpf_struct_ops_init(btf);
3641+
36083642
btf_verifier_env_free(env);
36093643
refcount_set(&btf->refcnt, 1);
36103644
return btf;

0 commit comments

Comments
 (0)