Skip to content

Commit 1c2a088

Browse files
4astborkmann
authored andcommitted
bpf: x64: add JIT support for multi-function programs
Typical JIT does several passes over bpf instructions to compute total size and relative offsets of jumps and calls. With multitple bpf functions calling each other all relative calls will have invalid offsets intially therefore we need to additional last pass over the program to emit calls with correct offsets. For example in case of three bpf functions: main: call foo call bpf_map_lookup exit foo: call bar exit bar: exit We will call bpf_int_jit_compile() indepedently for main(), foo() and bar() x64 JIT typically does 4-5 passes to converge. After these initial passes the image for these 3 functions will be good except call targets, since start addresses of foo() and bar() are unknown when we were JITing main() (note that call bpf_map_lookup will be resolved properly during initial passes). Once start addresses of 3 functions are known we patch call_insn->imm to point to right functions and call bpf_int_jit_compile() again which needs only one pass. Additional safety checks are done to make sure this last pass doesn't produce image that is larger or smaller than previous pass. When constant blinding is on it's applied to all functions at the first pass, since doing it once again at the last pass can change size of the JITed code. Tested on x64 and arm64 hw with JIT on/off, blinding on/off. x64 jits bpf-to-bpf calls correctly while arm64 falls back to interpreter. All other JITs that support normal BPF_CALL will behave the same way since bpf-to-bpf call is equivalent to bpf-to-kernel call from JITs point of view. Signed-off-by: Alexei Starovoitov <[email protected]> Acked-by: Daniel Borkmann <[email protected]> Signed-off-by: Daniel Borkmann <[email protected]>
1 parent 60b58af commit 1c2a088

File tree

7 files changed

+189
-6
lines changed

7 files changed

+189
-6
lines changed

arch/x86/net/bpf_jit_comp.c

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,13 +1109,23 @@ xadd: if (is_imm8(insn->off))
11091109
return proglen;
11101110
}
11111111

1112+
struct x64_jit_data {
1113+
struct bpf_binary_header *header;
1114+
int *addrs;
1115+
u8 *image;
1116+
int proglen;
1117+
struct jit_context ctx;
1118+
};
1119+
11121120
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
11131121
{
11141122
struct bpf_binary_header *header = NULL;
11151123
struct bpf_prog *tmp, *orig_prog = prog;
1124+
struct x64_jit_data *jit_data;
11161125
int proglen, oldproglen = 0;
11171126
struct jit_context ctx = {};
11181127
bool tmp_blinded = false;
1128+
bool extra_pass = false;
11191129
u8 *image = NULL;
11201130
int *addrs;
11211131
int pass;
@@ -1135,10 +1145,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
11351145
prog = tmp;
11361146
}
11371147

1148+
jit_data = prog->aux->jit_data;
1149+
if (!jit_data) {
1150+
jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1151+
if (!jit_data) {
1152+
prog = orig_prog;
1153+
goto out;
1154+
}
1155+
prog->aux->jit_data = jit_data;
1156+
}
1157+
addrs = jit_data->addrs;
1158+
if (addrs) {
1159+
ctx = jit_data->ctx;
1160+
oldproglen = jit_data->proglen;
1161+
image = jit_data->image;
1162+
header = jit_data->header;
1163+
extra_pass = true;
1164+
goto skip_init_addrs;
1165+
}
11381166
addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL);
11391167
if (!addrs) {
11401168
prog = orig_prog;
1141-
goto out;
1169+
goto out_addrs;
11421170
}
11431171

11441172
/* Before first pass, make a rough estimation of addrs[]
@@ -1149,6 +1177,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
11491177
addrs[i] = proglen;
11501178
}
11511179
ctx.cleanup_addr = proglen;
1180+
skip_init_addrs:
11521181

11531182
/* JITed image shrinks with every pass and the loop iterates
11541183
* until the image stops shrinking. Very large bpf programs
@@ -1189,16 +1218,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
11891218

11901219
if (image) {
11911220
bpf_flush_icache(header, image + proglen);
1192-
bpf_jit_binary_lock_ro(header);
1221+
if (!prog->is_func || extra_pass) {
1222+
bpf_jit_binary_lock_ro(header);
1223+
} else {
1224+
jit_data->addrs = addrs;
1225+
jit_data->ctx = ctx;
1226+
jit_data->proglen = proglen;
1227+
jit_data->image = image;
1228+
jit_data->header = header;
1229+
}
11931230
prog->bpf_func = (void *)image;
11941231
prog->jited = 1;
11951232
prog->jited_len = proglen;
11961233
} else {
11971234
prog = orig_prog;
11981235
}
11991236

1237+
if (!prog->is_func || extra_pass) {
12001238
out_addrs:
1201-
kfree(addrs);
1239+
kfree(addrs);
1240+
kfree(jit_data);
1241+
prog->aux->jit_data = NULL;
1242+
}
12021243
out:
12031244
if (tmp_blinded)
12041245
bpf_jit_prog_release_other(prog, prog == orig_prog ?

include/linux/bpf.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,9 @@ struct bpf_prog_aux {
200200
u32 max_ctx_offset;
201201
u32 stack_depth;
202202
u32 id;
203+
u32 func_cnt;
204+
struct bpf_prog **func;
205+
void *jit_data; /* JIT specific data. arch dependent */
203206
struct latch_tree_node ksym_tnode;
204207
struct list_head ksym_lnode;
205208
const struct bpf_prog_ops *ops;

include/linux/bpf_verifier.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ struct bpf_insn_aux_data {
143143
union {
144144
enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
145145
struct bpf_map *map_ptr; /* pointer for call insn into lookup_elem */
146+
s32 call_imm; /* saved imm field of call insn */
146147
};
147148
int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
148149
bool seen; /* this insn was processed by the verifier */

include/linux/filter.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,8 @@ struct bpf_prog {
463463
gpl_compatible:1, /* Is filter GPL compatible? */
464464
cb_access:1, /* Is control block accessed? */
465465
dst_needed:1, /* Do we need dst entry? */
466+
blinded:1, /* Was blinded */
467+
is_func:1, /* program is a bpf function */
466468
kprobe_override:1; /* Do we override a kprobe? */
467469
enum bpf_prog_type type; /* Type of BPF program */
468470
u32 len; /* Number of filter blocks */

kernel/bpf/core.c

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -722,7 +722,7 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
722722
struct bpf_insn *insn;
723723
int i, rewritten;
724724

725-
if (!bpf_jit_blinding_enabled(prog))
725+
if (!bpf_jit_blinding_enabled(prog) || prog->blinded)
726726
return prog;
727727

728728
clone = bpf_prog_clone_create(prog, GFP_USER);
@@ -764,6 +764,7 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
764764
i += insn_delta;
765765
}
766766

767+
clone->blinded = 1;
767768
return clone;
768769
}
769770
#endif /* CONFIG_BPF_JIT */
@@ -1629,11 +1630,19 @@ int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
16291630
static void bpf_prog_free_deferred(struct work_struct *work)
16301631
{
16311632
struct bpf_prog_aux *aux;
1633+
int i;
16321634

16331635
aux = container_of(work, struct bpf_prog_aux, work);
16341636
if (bpf_prog_is_dev_bound(aux))
16351637
bpf_prog_offload_destroy(aux->prog);
1636-
bpf_jit_free(aux->prog);
1638+
for (i = 0; i < aux->func_cnt; i++)
1639+
bpf_jit_free(aux->func[i]);
1640+
if (aux->func_cnt) {
1641+
kfree(aux->func);
1642+
bpf_prog_unlock_free(aux->prog);
1643+
} else {
1644+
bpf_jit_free(aux->prog);
1645+
}
16371646
}
16381647

16391648
/* Free internal BPF program */

kernel/bpf/syscall.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1194,7 +1194,8 @@ static int bpf_prog_load(union bpf_attr *attr)
11941194
goto free_used_maps;
11951195

11961196
/* eBPF program is ready to be JITed */
1197-
prog = bpf_prog_select_runtime(prog, &err);
1197+
if (!prog->bpf_func)
1198+
prog = bpf_prog_select_runtime(prog, &err);
11981199
if (err < 0)
11991200
goto free_used_maps;
12001201

kernel/bpf/verifier.c

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5012,12 +5012,138 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
50125012
return 0;
50135013
}
50145014

5015+
static int jit_subprogs(struct bpf_verifier_env *env)
5016+
{
5017+
struct bpf_prog *prog = env->prog, **func, *tmp;
5018+
int i, j, subprog_start, subprog_end = 0, len, subprog;
5019+
struct bpf_insn *insn = prog->insnsi;
5020+
void *old_bpf_func;
5021+
int err = -ENOMEM;
5022+
5023+
if (env->subprog_cnt == 0)
5024+
return 0;
5025+
5026+
for (i = 0; i < prog->len; i++, insn++) {
5027+
if (insn->code != (BPF_JMP | BPF_CALL) ||
5028+
insn->src_reg != BPF_PSEUDO_CALL)
5029+
continue;
5030+
subprog = find_subprog(env, i + insn->imm + 1);
5031+
if (subprog < 0) {
5032+
WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
5033+
i + insn->imm + 1);
5034+
return -EFAULT;
5035+
}
5036+
/* temporarily remember subprog id inside insn instead of
5037+
* aux_data, since next loop will split up all insns into funcs
5038+
*/
5039+
insn->off = subprog + 1;
5040+
/* remember original imm in case JIT fails and fallback
5041+
* to interpreter will be needed
5042+
*/
5043+
env->insn_aux_data[i].call_imm = insn->imm;
5044+
/* point imm to __bpf_call_base+1 from JITs point of view */
5045+
insn->imm = 1;
5046+
}
5047+
5048+
func = kzalloc(sizeof(prog) * (env->subprog_cnt + 1), GFP_KERNEL);
5049+
if (!func)
5050+
return -ENOMEM;
5051+
5052+
for (i = 0; i <= env->subprog_cnt; i++) {
5053+
subprog_start = subprog_end;
5054+
if (env->subprog_cnt == i)
5055+
subprog_end = prog->len;
5056+
else
5057+
subprog_end = env->subprog_starts[i];
5058+
5059+
len = subprog_end - subprog_start;
5060+
func[i] = bpf_prog_alloc(bpf_prog_size(len), GFP_USER);
5061+
if (!func[i])
5062+
goto out_free;
5063+
memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
5064+
len * sizeof(struct bpf_insn));
5065+
func[i]->len = len;
5066+
func[i]->is_func = 1;
5067+
/* Use bpf_prog_F_tag to indicate functions in stack traces.
5068+
* Long term would need debug info to populate names
5069+
*/
5070+
func[i]->aux->name[0] = 'F';
5071+
func[i]->aux->stack_depth = env->subprog_stack_depth[i];
5072+
func[i]->jit_requested = 1;
5073+
func[i] = bpf_int_jit_compile(func[i]);
5074+
if (!func[i]->jited) {
5075+
err = -ENOTSUPP;
5076+
goto out_free;
5077+
}
5078+
cond_resched();
5079+
}
5080+
/* at this point all bpf functions were successfully JITed
5081+
* now populate all bpf_calls with correct addresses and
5082+
* run last pass of JIT
5083+
*/
5084+
for (i = 0; i <= env->subprog_cnt; i++) {
5085+
insn = func[i]->insnsi;
5086+
for (j = 0; j < func[i]->len; j++, insn++) {
5087+
if (insn->code != (BPF_JMP | BPF_CALL) ||
5088+
insn->src_reg != BPF_PSEUDO_CALL)
5089+
continue;
5090+
subprog = insn->off;
5091+
insn->off = 0;
5092+
insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
5093+
func[subprog]->bpf_func -
5094+
__bpf_call_base;
5095+
}
5096+
}
5097+
for (i = 0; i <= env->subprog_cnt; i++) {
5098+
old_bpf_func = func[i]->bpf_func;
5099+
tmp = bpf_int_jit_compile(func[i]);
5100+
if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
5101+
verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
5102+
err = -EFAULT;
5103+
goto out_free;
5104+
}
5105+
cond_resched();
5106+
}
5107+
5108+
/* finally lock prog and jit images for all functions and
5109+
* populate kallsysm
5110+
*/
5111+
for (i = 0; i <= env->subprog_cnt; i++) {
5112+
bpf_prog_lock_ro(func[i]);
5113+
bpf_prog_kallsyms_add(func[i]);
5114+
}
5115+
prog->jited = 1;
5116+
prog->bpf_func = func[0]->bpf_func;
5117+
prog->aux->func = func;
5118+
prog->aux->func_cnt = env->subprog_cnt + 1;
5119+
return 0;
5120+
out_free:
5121+
for (i = 0; i <= env->subprog_cnt; i++)
5122+
if (func[i])
5123+
bpf_jit_free(func[i]);
5124+
kfree(func);
5125+
/* cleanup main prog to be interpreted */
5126+
prog->jit_requested = 0;
5127+
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
5128+
if (insn->code != (BPF_JMP | BPF_CALL) ||
5129+
insn->src_reg != BPF_PSEUDO_CALL)
5130+
continue;
5131+
insn->off = 0;
5132+
insn->imm = env->insn_aux_data[i].call_imm;
5133+
}
5134+
return err;
5135+
}
5136+
50155137
static int fixup_call_args(struct bpf_verifier_env *env)
50165138
{
50175139
struct bpf_prog *prog = env->prog;
50185140
struct bpf_insn *insn = prog->insnsi;
50195141
int i, depth;
50205142

5143+
if (env->prog->jit_requested)
5144+
if (jit_subprogs(env) == 0)
5145+
return 0;
5146+
50215147
for (i = 0; i < prog->len; i++, insn++) {
50225148
if (insn->code != (BPF_JMP | BPF_CALL) ||
50235149
insn->src_reg != BPF_PSEUDO_CALL)

0 commit comments

Comments
 (0)