Skip to content

Commit 8008342

Browse files
jpbruckerborkmann
authored andcommitted
bpf, arm64: Add BPF exception tables
When a tracing BPF program attempts to read memory without using the bpf_probe_read() helper, the verifier marks the load instruction with the BPF_PROBE_MEM flag. Since the arm64 JIT does not currently recognize this flag it falls back to the interpreter. Add support for BPF_PROBE_MEM, by appending an exception table to the BPF program. If the load instruction causes a data abort, the fixup infrastructure finds the exception table and fixes up the fault, by clearing the destination register and jumping over the faulting instruction. To keep the compact exception table entry format, inspect the pc in fixup_exception(). A more generic solution would add a "handler" field to the table entry, like on x86 and s390. Signed-off-by: Jean-Philippe Brucker <[email protected]> Signed-off-by: Daniel Borkmann <[email protected]> Acked-by: Song Liu <[email protected]> Link: https://lore.kernel.org/bpf/[email protected]
1 parent 310ad79 commit 8008342

File tree

3 files changed

+108
-9
lines changed

3 files changed

+108
-9
lines changed

arch/arm64/include/asm/extable.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,17 @@ struct exception_table_entry
2222

2323
#define ARCH_HAS_RELATIVE_EXTABLE
2424

25+
#ifdef CONFIG_BPF_JIT
26+
int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
27+
struct pt_regs *regs);
28+
#else /* !CONFIG_BPF_JIT */
29+
static inline
30+
int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
31+
struct pt_regs *regs)
32+
{
33+
return 0;
34+
}
35+
#endif /* !CONFIG_BPF_JIT */
36+
2537
extern int fixup_exception(struct pt_regs *regs);
2638
#endif

arch/arm64/mm/extable.c

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,14 @@ int fixup_exception(struct pt_regs *regs)
1111
const struct exception_table_entry *fixup;
1212

1313
fixup = search_exception_tables(instruction_pointer(regs));
14-
if (fixup)
15-
regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
14+
if (!fixup)
15+
return 0;
1616

17-
return fixup != NULL;
17+
if (IS_ENABLED(CONFIG_BPF_JIT) &&
18+
regs->pc >= BPF_JIT_REGION_START &&
19+
regs->pc < BPF_JIT_REGION_END)
20+
return arm64_bpf_fixup_exception(fixup, regs);
21+
22+
regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
23+
return 1;
1824
}

arch/arm64/net/bpf_jit_comp.c

Lines changed: 87 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
#define pr_fmt(fmt) "bpf_jit: " fmt
99

10+
#include <linux/bitfield.h>
1011
#include <linux/bpf.h>
1112
#include <linux/filter.h>
1213
#include <linux/printk.h>
@@ -56,6 +57,7 @@ struct jit_ctx {
5657
int idx;
5758
int epilogue_offset;
5859
int *offset;
60+
int exentry_idx;
5961
__le32 *image;
6062
u32 stack_size;
6163
};
@@ -351,6 +353,67 @@ static void build_epilogue(struct jit_ctx *ctx)
351353
emit(A64_RET(A64_LR), ctx);
352354
}
353355

356+
#define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
357+
#define BPF_FIXUP_REG_MASK GENMASK(31, 27)
358+
359+
int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
360+
struct pt_regs *regs)
361+
{
362+
off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
363+
int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
364+
365+
regs->regs[dst_reg] = 0;
366+
regs->pc = (unsigned long)&ex->fixup - offset;
367+
return 1;
368+
}
369+
370+
/* For accesses to BTF pointers, add an entry to the exception table */
371+
static int add_exception_handler(const struct bpf_insn *insn,
372+
struct jit_ctx *ctx,
373+
int dst_reg)
374+
{
375+
off_t offset;
376+
unsigned long pc;
377+
struct exception_table_entry *ex;
378+
379+
if (!ctx->image)
380+
/* First pass */
381+
return 0;
382+
383+
if (BPF_MODE(insn->code) != BPF_PROBE_MEM)
384+
return 0;
385+
386+
if (!ctx->prog->aux->extable ||
387+
WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries))
388+
return -EINVAL;
389+
390+
ex = &ctx->prog->aux->extable[ctx->exentry_idx];
391+
pc = (unsigned long)&ctx->image[ctx->idx - 1];
392+
393+
offset = pc - (long)&ex->insn;
394+
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
395+
return -ERANGE;
396+
ex->insn = offset;
397+
398+
/*
399+
* Since the extable follows the program, the fixup offset is always
400+
* negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
401+
* to keep things simple, and put the destination register in the upper
402+
* bits. We don't need to worry about buildtime or runtime sort
403+
* modifying the upper bits because the table is already sorted, and
404+
* isn't part of the main exception table.
405+
*/
406+
offset = (long)&ex->fixup - (pc + AARCH64_INSN_SIZE);
407+
if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
408+
return -ERANGE;
409+
410+
ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
411+
FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
412+
413+
ctx->exentry_idx++;
414+
return 0;
415+
}
416+
354417
/* JITs an eBPF instruction.
355418
* Returns:
356419
* 0 - successfully JITed an 8-byte eBPF instruction.
@@ -375,6 +438,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
375438
u8 jmp_cond, reg;
376439
s32 jmp_offset;
377440
u32 a64_insn;
441+
int ret;
378442

379443
#define check_imm(bits, imm) do { \
380444
if ((((imm) > 0) && ((imm) >> (bits))) || \
@@ -694,7 +758,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
694758
const u8 r0 = bpf2a64[BPF_REG_0];
695759
bool func_addr_fixed;
696760
u64 func_addr;
697-
int ret;
698761

699762
ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
700763
&func_addr, &func_addr_fixed);
@@ -738,6 +801,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
738801
case BPF_LDX | BPF_MEM | BPF_H:
739802
case BPF_LDX | BPF_MEM | BPF_B:
740803
case BPF_LDX | BPF_MEM | BPF_DW:
804+
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
805+
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
806+
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
807+
case BPF_LDX | BPF_PROBE_MEM | BPF_B:
741808
emit_a64_mov_i(1, tmp, off, ctx);
742809
switch (BPF_SIZE(code)) {
743810
case BPF_W:
@@ -753,6 +820,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
753820
emit(A64_LDR64(dst, src, tmp), ctx);
754821
break;
755822
}
823+
824+
ret = add_exception_handler(insn, ctx, dst);
825+
if (ret)
826+
return ret;
756827
break;
757828

758829
/* ST: *(size *)(dst + off) = imm */
@@ -868,6 +939,9 @@ static int validate_code(struct jit_ctx *ctx)
868939
return -1;
869940
}
870941

942+
if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries))
943+
return -1;
944+
871945
return 0;
872946
}
873947

@@ -884,14 +958,14 @@ struct arm64_jit_data {
884958

885959
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
886960
{
961+
int image_size, prog_size, extable_size;
887962
struct bpf_prog *tmp, *orig_prog = prog;
888963
struct bpf_binary_header *header;
889964
struct arm64_jit_data *jit_data;
890965
bool was_classic = bpf_prog_was_classic(prog);
891966
bool tmp_blinded = false;
892967
bool extra_pass = false;
893968
struct jit_ctx ctx;
894-
int image_size;
895969
u8 *image_ptr;
896970

897971
if (!prog->jit_requested)
@@ -922,7 +996,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
922996
image_ptr = jit_data->image;
923997
header = jit_data->header;
924998
extra_pass = true;
925-
image_size = sizeof(u32) * ctx.idx;
999+
prog_size = sizeof(u32) * ctx.idx;
9261000
goto skip_init_ctx;
9271001
}
9281002
memset(&ctx, 0, sizeof(ctx));
@@ -950,8 +1024,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
9501024
ctx.epilogue_offset = ctx.idx;
9511025
build_epilogue(&ctx);
9521026

1027+
extable_size = prog->aux->num_exentries *
1028+
sizeof(struct exception_table_entry);
1029+
9531030
/* Now we know the actual image size. */
954-
image_size = sizeof(u32) * ctx.idx;
1031+
prog_size = sizeof(u32) * ctx.idx;
1032+
image_size = prog_size + extable_size;
9551033
header = bpf_jit_binary_alloc(image_size, &image_ptr,
9561034
sizeof(u32), jit_fill_hole);
9571035
if (header == NULL) {
@@ -962,8 +1040,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
9621040
/* 2. Now, the actual pass. */
9631041

9641042
ctx.image = (__le32 *)image_ptr;
1043+
if (extable_size)
1044+
prog->aux->extable = (void *)image_ptr + prog_size;
9651045
skip_init_ctx:
9661046
ctx.idx = 0;
1047+
ctx.exentry_idx = 0;
9671048

9681049
build_prologue(&ctx, was_classic);
9691050

@@ -984,7 +1065,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
9841065

9851066
/* And we're done. */
9861067
if (bpf_jit_enable > 1)
987-
bpf_jit_dump(prog->len, image_size, 2, ctx.image);
1068+
bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
9881069

9891070
bpf_flush_icache(header, ctx.image + ctx.idx);
9901071

@@ -1005,7 +1086,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
10051086
}
10061087
prog->bpf_func = (void *)ctx.image;
10071088
prog->jited = 1;
1008-
prog->jited_len = image_size;
1089+
prog->jited_len = prog_size;
10091090

10101091
if (!prog->is_func || extra_pass) {
10111092
bpf_prog_fill_jited_linfo(prog, ctx.offset);

0 commit comments

Comments
 (0)