Skip to content

Commit 500322e

Browse files
davem330davem330
committed
Merge branch 'bpf-push-pop-helpers'
Alexei Starovoitov says: ==================== bpf: introduce bpf_skb_vlan_push/pop() helpers Let TC+eBPF programs call skb_vlan_push/pop via helpers. v1->v2: - reworded commit log to better explain correctness of re-caching and fixed comparison of mixed endiannes (suggested by Eric) ==================== Signed-off-by: David S. Miller <[email protected]>
2 parents f3120ac + 4d9c5c5 commit 500322e

File tree

8 files changed

+197
-41
lines changed

8 files changed

+197
-41
lines changed

‎arch/s390/net/bpf_jit_comp.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -973,6 +973,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
973973
*/
974974
const u64 func = (u64)__bpf_call_base + imm;
975975

976+
if (bpf_helper_changes_skb_data((void *)func))
977+
/* TODO reload skb->data, hlen */
978+
return -1;
979+
976980
REG_SET_SEEN(BPF_REG_5);
977981
jit->seen |= SEEN_FUNC;
978982
/* lg %w1,<d(imm)>(%l) */

‎arch/x86/net/bpf_jit_comp.c

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,26 @@ static void emit_bpf_tail_call(u8 **pprog)
315315
*pprog = prog;
316316
}
317317

318+
319+
static void emit_load_skb_data_hlen(u8 **pprog)
320+
{
321+
u8 *prog = *pprog;
322+
int cnt = 0;
323+
324+
/* r9d = skb->len - skb->data_len (headlen)
325+
* r10 = skb->data
326+
*/
327+
/* mov %r9d, off32(%rdi) */
328+
EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len));
329+
330+
/* sub %r9d, off32(%rdi) */
331+
EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len));
332+
333+
/* mov %r10, off32(%rdi) */
334+
EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data));
335+
*pprog = prog;
336+
}
337+
318338
static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
319339
int oldproglen, struct jit_context *ctx)
320340
{
@@ -329,36 +349,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
329349

330350
emit_prologue(&prog);
331351

332-
if (seen_ld_abs) {
333-
/* r9d : skb->len - skb->data_len (headlen)
334-
* r10 : skb->data
335-
*/
336-
if (is_imm8(offsetof(struct sk_buff, len)))
337-
/* mov %r9d, off8(%rdi) */
338-
EMIT4(0x44, 0x8b, 0x4f,
339-
offsetof(struct sk_buff, len));
340-
else
341-
/* mov %r9d, off32(%rdi) */
342-
EMIT3_off32(0x44, 0x8b, 0x8f,
343-
offsetof(struct sk_buff, len));
344-
345-
if (is_imm8(offsetof(struct sk_buff, data_len)))
346-
/* sub %r9d, off8(%rdi) */
347-
EMIT4(0x44, 0x2b, 0x4f,
348-
offsetof(struct sk_buff, data_len));
349-
else
350-
EMIT3_off32(0x44, 0x2b, 0x8f,
351-
offsetof(struct sk_buff, data_len));
352-
353-
if (is_imm8(offsetof(struct sk_buff, data)))
354-
/* mov %r10, off8(%rdi) */
355-
EMIT4(0x4c, 0x8b, 0x57,
356-
offsetof(struct sk_buff, data));
357-
else
358-
/* mov %r10, off32(%rdi) */
359-
EMIT3_off32(0x4c, 0x8b, 0x97,
360-
offsetof(struct sk_buff, data));
361-
}
352+
if (seen_ld_abs)
353+
emit_load_skb_data_hlen(&prog);
362354

363355
for (i = 0; i < insn_cnt; i++, insn++) {
364356
const s32 imm32 = insn->imm;
@@ -367,6 +359,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
367359
u8 b1 = 0, b2 = 0, b3 = 0;
368360
s64 jmp_offset;
369361
u8 jmp_cond;
362+
bool reload_skb_data;
370363
int ilen;
371364
u8 *func;
372365

@@ -818,12 +811,18 @@ xadd: if (is_imm8(insn->off))
818811
func = (u8 *) __bpf_call_base + imm32;
819812
jmp_offset = func - (image + addrs[i]);
820813
if (seen_ld_abs) {
821-
EMIT2(0x41, 0x52); /* push %r10 */
822-
EMIT2(0x41, 0x51); /* push %r9 */
823-
/* need to adjust jmp offset, since
824-
* pop %r9, pop %r10 take 4 bytes after call insn
825-
*/
826-
jmp_offset += 4;
814+
reload_skb_data = bpf_helper_changes_skb_data(func);
815+
if (reload_skb_data) {
816+
EMIT1(0x57); /* push %rdi */
817+
jmp_offset += 22; /* pop, mov, sub, mov */
818+
} else {
819+
EMIT2(0x41, 0x52); /* push %r10 */
820+
EMIT2(0x41, 0x51); /* push %r9 */
821+
/* need to adjust jmp offset, since
822+
* pop %r9, pop %r10 take 4 bytes after call insn
823+
*/
824+
jmp_offset += 4;
825+
}
827826
}
828827
if (!imm32 || !is_simm32(jmp_offset)) {
829828
pr_err("unsupported bpf func %d addr %p image %p\n",
@@ -832,8 +831,13 @@ xadd: if (is_imm8(insn->off))
832831
}
833832
EMIT1_off32(0xE8, jmp_offset);
834833
if (seen_ld_abs) {
835-
EMIT2(0x41, 0x59); /* pop %r9 */
836-
EMIT2(0x41, 0x5A); /* pop %r10 */
834+
if (reload_skb_data) {
835+
EMIT1(0x5F); /* pop %rdi */
836+
emit_load_skb_data_hlen(&prog);
837+
} else {
838+
EMIT2(0x41, 0x59); /* pop %r9 */
839+
EMIT2(0x41, 0x5A); /* pop %r10 */
840+
}
837841
}
838842
break;
839843

‎include/linux/bpf.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,5 +192,7 @@ extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
192192
extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
193193
extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
194194
extern const struct bpf_func_proto bpf_get_current_comm_proto;
195+
extern const struct bpf_func_proto bpf_skb_vlan_push_proto;
196+
extern const struct bpf_func_proto bpf_skb_vlan_pop_proto;
195197

196198
#endif /* _LINUX_BPF_H */

‎include/linux/filter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,7 @@ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
411411

412412
u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
413413
void bpf_int_jit_compile(struct bpf_prog *fp);
414+
bool bpf_helper_changes_skb_data(void *func);
414415

415416
#ifdef CONFIG_BPF_JIT
416417
typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);

‎include/uapi/linux/bpf.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,8 @@ enum bpf_func_id {
256256
* Return: classid if != 0
257257
*/
258258
BPF_FUNC_get_cgroup_classid,
259+
BPF_FUNC_skb_vlan_push, /* bpf_skb_vlan_push(skb, vlan_proto, vlan_tci) */
260+
BPF_FUNC_skb_vlan_pop, /* bpf_skb_vlan_pop(skb) */
259261
__BPF_FUNC_MAX_ID,
260262
};
261263

‎kernel/bpf/core.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
177177
{
178178
return 0;
179179
}
180+
EXPORT_SYMBOL_GPL(__bpf_call_base);
180181

181182
/**
182183
* __bpf_prog_run - run eBPF program on a given context

‎lib/test_bpf.c

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include <linux/init.h>
1919
#include <linux/module.h>
2020
#include <linux/filter.h>
21+
#include <linux/bpf.h>
2122
#include <linux/skbuff.h>
2223
#include <linux/netdevice.h>
2324
#include <linux/if_vlan.h>
@@ -355,6 +356,81 @@ static int bpf_fill_ja(struct bpf_test *self)
355356
return __bpf_fill_ja(self, 12, 9);
356357
}
357358

359+
static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self)
360+
{
361+
unsigned int len = BPF_MAXINSNS;
362+
struct sock_filter *insn;
363+
int i;
364+
365+
insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
366+
if (!insn)
367+
return -ENOMEM;
368+
369+
for (i = 0; i < len - 1; i += 2) {
370+
insn[i] = __BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 0);
371+
insn[i + 1] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS,
372+
SKF_AD_OFF + SKF_AD_CPU);
373+
}
374+
375+
insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xbee);
376+
377+
self->u.ptr.insns = insn;
378+
self->u.ptr.len = len;
379+
380+
return 0;
381+
}
382+
383+
#define PUSH_CNT 68
384+
/* test: {skb->data[0], vlan_push} x 68 + {skb->data[0], vlan_pop} x 68 */
385+
static int bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
386+
{
387+
unsigned int len = BPF_MAXINSNS;
388+
struct bpf_insn *insn;
389+
int i = 0, j, k = 0;
390+
391+
insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
392+
if (!insn)
393+
return -ENOMEM;
394+
395+
insn[i++] = BPF_MOV64_REG(R6, R1);
396+
loop:
397+
for (j = 0; j < PUSH_CNT; j++) {
398+
insn[i++] = BPF_LD_ABS(BPF_B, 0);
399+
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0x34, len - i - 2);
400+
i++;
401+
insn[i++] = BPF_MOV64_REG(R1, R6);
402+
insn[i++] = BPF_MOV64_IMM(R2, 1);
403+
insn[i++] = BPF_MOV64_IMM(R3, 2);
404+
insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
405+
bpf_skb_vlan_push_proto.func - __bpf_call_base);
406+
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0, len - i - 2);
407+
i++;
408+
}
409+
410+
for (j = 0; j < PUSH_CNT; j++) {
411+
insn[i++] = BPF_LD_ABS(BPF_B, 0);
412+
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0x34, len - i - 2);
413+
i++;
414+
insn[i++] = BPF_MOV64_REG(R1, R6);
415+
insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
416+
bpf_skb_vlan_pop_proto.func - __bpf_call_base);
417+
insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0, len - i - 2);
418+
i++;
419+
}
420+
if (++k < 5)
421+
goto loop;
422+
423+
for (; i < len - 1; i++)
424+
insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xbef);
425+
426+
insn[len - 1] = BPF_EXIT_INSN();
427+
428+
self->u.ptr.insns = insn;
429+
self->u.ptr.len = len;
430+
431+
return 0;
432+
}
433+
358434
static struct bpf_test tests[] = {
359435
{
360436
"TAX",
@@ -4398,6 +4474,22 @@ static struct bpf_test tests[] = {
43984474
{ { 0, 0xababcbac } },
43994475
.fill_helper = bpf_fill_maxinsns11,
44004476
},
4477+
{
4478+
"BPF_MAXINSNS: ld_abs+get_processor_id",
4479+
{ },
4480+
CLASSIC,
4481+
{ },
4482+
{ { 1, 0xbee } },
4483+
.fill_helper = bpf_fill_ld_abs_get_processor_id,
4484+
},
4485+
{
4486+
"BPF_MAXINSNS: ld_abs+vlan_push/pop",
4487+
{ },
4488+
INTERNAL,
4489+
{ 0x34 },
4490+
{ { 1, 0xbef } },
4491+
.fill_helper = bpf_fill_ld_abs_vlan_push_pop,
4492+
},
44014493
};
44024494

44034495
static struct net_device dev;
@@ -4551,14 +4643,14 @@ static int __run_one(const struct bpf_prog *fp, const void *data,
45514643
u64 start, finish;
45524644
int ret = 0, i;
45534645

4554-
start = ktime_to_us(ktime_get());
4646+
start = ktime_get_ns();
45554647

45564648
for (i = 0; i < runs; i++)
45574649
ret = BPF_PROG_RUN(fp, data);
45584650

4559-
finish = ktime_to_us(ktime_get());
4651+
finish = ktime_get_ns();
45604652

4561-
*duration = (finish - start) * 1000ULL;
4653+
*duration = finish - start;
45624654
do_div(*duration, runs);
45634655

45644656
return ret;

‎net/core/filter.c

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1437,6 +1437,52 @@ static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
14371437
.arg1_type = ARG_PTR_TO_CTX,
14381438
};
14391439

1440+
static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5)
1441+
{
1442+
struct sk_buff *skb = (struct sk_buff *) (long) r1;
1443+
__be16 vlan_proto = (__force __be16) r2;
1444+
1445+
if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
1446+
vlan_proto != htons(ETH_P_8021AD)))
1447+
vlan_proto = htons(ETH_P_8021Q);
1448+
1449+
return skb_vlan_push(skb, vlan_proto, vlan_tci);
1450+
}
1451+
1452+
const struct bpf_func_proto bpf_skb_vlan_push_proto = {
1453+
.func = bpf_skb_vlan_push,
1454+
.gpl_only = false,
1455+
.ret_type = RET_INTEGER,
1456+
.arg1_type = ARG_PTR_TO_CTX,
1457+
.arg2_type = ARG_ANYTHING,
1458+
.arg3_type = ARG_ANYTHING,
1459+
};
1460+
EXPORT_SYMBOL_GPL(bpf_skb_vlan_push_proto);
1461+
1462+
static u64 bpf_skb_vlan_pop(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
1463+
{
1464+
struct sk_buff *skb = (struct sk_buff *) (long) r1;
1465+
1466+
return skb_vlan_pop(skb);
1467+
}
1468+
1469+
const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
1470+
.func = bpf_skb_vlan_pop,
1471+
.gpl_only = false,
1472+
.ret_type = RET_INTEGER,
1473+
.arg1_type = ARG_PTR_TO_CTX,
1474+
};
1475+
EXPORT_SYMBOL_GPL(bpf_skb_vlan_pop_proto);
1476+
1477+
bool bpf_helper_changes_skb_data(void *func)
1478+
{
1479+
if (func == bpf_skb_vlan_push)
1480+
return true;
1481+
if (func == bpf_skb_vlan_pop)
1482+
return true;
1483+
return false;
1484+
}
1485+
14401486
static const struct bpf_func_proto *
14411487
sk_filter_func_proto(enum bpf_func_id func_id)
14421488
{
@@ -1476,6 +1522,10 @@ tc_cls_act_func_proto(enum bpf_func_id func_id)
14761522
return &bpf_clone_redirect_proto;
14771523
case BPF_FUNC_get_cgroup_classid:
14781524
return &bpf_get_cgroup_classid_proto;
1525+
case BPF_FUNC_skb_vlan_push:
1526+
return &bpf_skb_vlan_push_proto;
1527+
case BPF_FUNC_skb_vlan_pop:
1528+
return &bpf_skb_vlan_pop_proto;
14791529
default:
14801530
return sk_filter_func_proto(func_id);
14811531
}

0 commit comments

Comments
 (0)