Skip to content

Commit b991fc5

Browse files
eddyz87Alexei Starovoitov
authored andcommitted
selftests/bpf: utility function to get program disassembly after jit
This commit adds a utility function to get disassembled text for jited representation of a BPF program designated by file descriptor. Function prototype looks as follows: int get_jited_program_text(int fd, char *text, size_t text_sz) Where 'fd' is a file descriptor for the program, 'text' and 'text_sz' refer to a destination buffer for disassembled text. Output format looks as follows: 18: 77 06 ja L0 1a: 50 pushq %rax 1b: 48 89 e0 movq %rsp, %rax 1e: eb 01 jmp L1 20: 50 L0: pushq %rax 21: 50 L1: pushq %rax ^ ^^^^^^^^ ^ ^^^^^^^^^^^^^^^^^^ | binary insn | textual insn | representation | representation | | instruction offset inferred local label name The code and makefile changes are inspired by jit_disasm.c from bpftool. Use llvm libraries to disassemble BPF program instead of libbfd to avoid issues with disassembly output stability pointed out in [1]. Selftests makefile uses Makefile.feature to detect if LLVM libraries are available. If that is not the case selftests build proceeds but the function returns -EOPNOTSUPP at runtime. [1] commit eb9d1ac ("bpftool: Add LLVM as default library for disassembling JIT-ed programs") Acked-by: Yonghong Song <[email protected]> Signed-off-by: Eduard Zingerman <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Alexei Starovoitov <[email protected]>
1 parent f8d1617 commit b991fc5

File tree

4 files changed

+291
-2
lines changed

4 files changed

+291
-2
lines changed

tools/testing/selftests/bpf/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ test_lru_map
88
test_lpm_map
99
test_tag
1010
FEATURE-DUMP.libbpf
11+
FEATURE-DUMP.selftests
1112
fixdep
1213
/test_progs
1314
/test_progs-no_alu32

tools/testing/selftests/bpf/Makefile

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ OPT_FLAGS ?= $(if $(RELEASE),-O2,-O0)
3333
LIBELF_CFLAGS := $(shell $(PKG_CONFIG) libelf --cflags 2>/dev/null)
3434
LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
3535

36+
ifeq ($(srctree),)
37+
srctree := $(patsubst %/,%,$(dir $(CURDIR)))
38+
srctree := $(patsubst %/,%,$(dir $(srctree)))
39+
srctree := $(patsubst %/,%,$(dir $(srctree)))
40+
srctree := $(patsubst %/,%,$(dir $(srctree)))
41+
endif
42+
3643
CFLAGS += -g $(OPT_FLAGS) -rdynamic \
3744
-Wall -Werror -fno-omit-frame-pointer \
3845
$(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \
@@ -60,6 +67,9 @@ progs/timer_crash.c-CFLAGS := -fno-strict-aliasing
6067
progs/test_global_func9.c-CFLAGS := -fno-strict-aliasing
6168
progs/verifier_nocsr.c-CFLAGS := -fno-strict-aliasing
6269

70+
# Some utility functions use LLVM libraries
71+
jit_disasm_helpers.c-CFLAGS = $(LLVM_CFLAGS)
72+
6373
ifneq ($(LLVM),)
6474
# Silence some warnings when compiled with clang
6575
CFLAGS += -Wno-unused-command-line-argument
@@ -168,6 +178,31 @@ endef
168178

169179
include ../lib.mk
170180

181+
NON_CHECK_FEAT_TARGETS := clean docs-clean
182+
CHECK_FEAT := $(filter-out $(NON_CHECK_FEAT_TARGETS),$(or $(MAKECMDGOALS), "none"))
183+
ifneq ($(CHECK_FEAT),)
184+
FEATURE_USER := .selftests
185+
FEATURE_TESTS := llvm
186+
FEATURE_DISPLAY := $(FEATURE_TESTS)
187+
188+
# Makefile.feature expects OUTPUT to end with a slash
189+
$(let OUTPUT,$(OUTPUT)/,\
190+
$(eval include ../../../build/Makefile.feature))
191+
endif
192+
193+
ifeq ($(feature-llvm),1)
194+
LLVM_CFLAGS += -DHAVE_LLVM_SUPPORT
195+
LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
196+
# both llvm-config and lib.mk add -D_GNU_SOURCE, which ends up as conflict
197+
LLVM_CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
198+
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
199+
ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)
200+
LLVM_LDLIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
201+
LLVM_LDLIBS += -lstdc++
202+
endif
203+
LLVM_LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
204+
endif
205+
171206
SCRATCH_DIR := $(OUTPUT)/tools
172207
BUILD_DIR := $(SCRATCH_DIR)/build
173208
INCLUDE_DIR := $(SCRATCH_DIR)/include
@@ -612,6 +647,10 @@ ifeq ($(filter clean docs-clean,$(MAKECMDGOALS)),)
612647
include $(wildcard $(TRUNNER_TEST_OBJS:.o=.d))
613648
endif
614649

650+
# add per extra obj CFGLAGS definitions
651+
$(foreach N,$(patsubst $(TRUNNER_OUTPUT)/%.o,%,$(TRUNNER_EXTRA_OBJS)), \
652+
$(eval $(TRUNNER_OUTPUT)/$(N).o: CFLAGS += $($(N).c-CFLAGS)))
653+
615654
$(TRUNNER_EXTRA_OBJS): $(TRUNNER_OUTPUT)/%.o: \
616655
%.c \
617656
$(TRUNNER_EXTRA_HDRS) \
@@ -628,6 +667,9 @@ ifneq ($2:$(OUTPUT),:$(shell pwd))
628667
$(Q)rsync -aq $$^ $(TRUNNER_OUTPUT)/
629668
endif
630669

670+
$(OUTPUT)/$(TRUNNER_BINARY): LDLIBS += $$(LLVM_LDLIBS)
671+
$(OUTPUT)/$(TRUNNER_BINARY): LDFLAGS += $$(LLVM_LDFLAGS)
672+
631673
# some X.test.o files have runtime dependencies on Y.bpf.o files
632674
$(OUTPUT)/$(TRUNNER_BINARY): | $(TRUNNER_BPF_OBJS)
633675

@@ -637,7 +679,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \
637679
$(TRUNNER_BPFTOOL) \
638680
| $(TRUNNER_BINARY)-extras
639681
$$(call msg,BINARY,,$$@)
640-
$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) -o $$@
682+
$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LDFLAGS) -o $$@
641683
$(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@
642684
$(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/$(USE_BOOTSTRAP)bpftool \
643685
$(OUTPUT)/$(if $2,$2/)bpftool
@@ -656,6 +698,7 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \
656698
cap_helpers.c \
657699
unpriv_helpers.c \
658700
netlink_helpers.c \
701+
jit_disasm_helpers.c \
659702
test_loader.c \
660703
xsk.c \
661704
disasm.c \
@@ -798,7 +841,8 @@ EXTRA_CLEAN := $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \
798841
$(addprefix $(OUTPUT)/,*.o *.d *.skel.h *.lskel.h *.subskel.h \
799842
no_alu32 cpuv4 bpf_gcc bpf_testmod.ko \
800843
bpf_test_no_cfi.ko \
801-
liburandom_read.so)
844+
liburandom_read.so) \
845+
$(OUTPUT)/FEATURE-DUMP.selftests
802846

803847
.PHONY: docs docs-clean
804848

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2+
#include <bpf/bpf.h>
3+
#include <bpf/libbpf.h>
4+
#include <test_progs.h>
5+
6+
#ifdef HAVE_LLVM_SUPPORT
7+
8+
#include <llvm-c/Core.h>
9+
#include <llvm-c/Disassembler.h>
10+
#include <llvm-c/Target.h>
11+
#include <llvm-c/TargetMachine.h>
12+
13+
/* The intent is to use get_jited_program_text() for small test
14+
* programs written in BPF assembly, thus assume that 32 local labels
15+
* would be sufficient.
16+
*/
17+
#define MAX_LOCAL_LABELS 32
18+
19+
static bool llvm_initialized;
20+
21+
struct local_labels {
22+
bool print_phase;
23+
__u32 prog_len;
24+
__u32 cnt;
25+
__u32 pcs[MAX_LOCAL_LABELS];
26+
char names[MAX_LOCAL_LABELS][4];
27+
};
28+
29+
static const char *lookup_symbol(void *data, uint64_t ref_value, uint64_t *ref_type,
30+
uint64_t ref_pc, const char **ref_name)
31+
{
32+
struct local_labels *labels = data;
33+
uint64_t type = *ref_type;
34+
int i;
35+
36+
*ref_type = LLVMDisassembler_ReferenceType_InOut_None;
37+
*ref_name = NULL;
38+
if (type != LLVMDisassembler_ReferenceType_In_Branch)
39+
return NULL;
40+
/* Depending on labels->print_phase either discover local labels or
41+
* return a name assigned with local jump target:
42+
* - if print_phase is true and ref_value is in labels->pcs,
43+
* return corresponding labels->name.
44+
* - if print_phase is false, save program-local jump targets
45+
* in labels->pcs;
46+
*/
47+
if (labels->print_phase) {
48+
for (i = 0; i < labels->cnt; ++i)
49+
if (labels->pcs[i] == ref_value)
50+
return labels->names[i];
51+
} else {
52+
if (labels->cnt < MAX_LOCAL_LABELS && ref_value < labels->prog_len)
53+
labels->pcs[labels->cnt++] = ref_value;
54+
}
55+
return NULL;
56+
}
57+
58+
static int disasm_insn(LLVMDisasmContextRef ctx, uint8_t *image, __u32 len, __u32 pc,
59+
char *buf, __u32 buf_sz)
60+
{
61+
int i, cnt;
62+
63+
cnt = LLVMDisasmInstruction(ctx, image + pc, len - pc, pc,
64+
buf, buf_sz);
65+
if (cnt > 0)
66+
return cnt;
67+
PRINT_FAIL("Can't disasm instruction at offset %d:", pc);
68+
for (i = 0; i < 16 && pc + i < len; ++i)
69+
printf(" %02x", image[pc + i]);
70+
printf("\n");
71+
return -EINVAL;
72+
}
73+
74+
static int cmp_u32(const void *_a, const void *_b)
75+
{
76+
__u32 a = *(__u32 *)_a;
77+
__u32 b = *(__u32 *)_b;
78+
79+
if (a < b)
80+
return -1;
81+
if (a > b)
82+
return 1;
83+
return 0;
84+
}
85+
86+
static int disasm_one_func(FILE *text_out, uint8_t *image, __u32 len)
87+
{
88+
char *label, *colon, *triple = NULL;
89+
LLVMDisasmContextRef ctx = NULL;
90+
struct local_labels labels = {};
91+
__u32 *label_pc, pc;
92+
int i, cnt, err = 0;
93+
char buf[64];
94+
95+
triple = LLVMGetDefaultTargetTriple();
96+
ctx = LLVMCreateDisasm(triple, &labels, 0, NULL, lookup_symbol);
97+
if (!ASSERT_OK_PTR(ctx, "LLVMCreateDisasm")) {
98+
err = -EINVAL;
99+
goto out;
100+
}
101+
102+
cnt = LLVMSetDisasmOptions(ctx, LLVMDisassembler_Option_PrintImmHex);
103+
if (!ASSERT_EQ(cnt, 1, "LLVMSetDisasmOptions")) {
104+
err = -EINVAL;
105+
goto out;
106+
}
107+
108+
/* discover labels */
109+
labels.prog_len = len;
110+
pc = 0;
111+
while (pc < len) {
112+
cnt = disasm_insn(ctx, image, len, pc, buf, 1);
113+
if (cnt < 0) {
114+
err = cnt;
115+
goto out;
116+
}
117+
pc += cnt;
118+
}
119+
qsort(labels.pcs, labels.cnt, sizeof(*labels.pcs), cmp_u32);
120+
for (i = 0; i < labels.cnt; ++i)
121+
/* use (i % 100) to avoid format truncation warning */
122+
snprintf(labels.names[i], sizeof(labels.names[i]), "L%d", i % 100);
123+
124+
/* now print with labels */
125+
labels.print_phase = true;
126+
pc = 0;
127+
while (pc < len) {
128+
cnt = disasm_insn(ctx, image, len, pc, buf, sizeof(buf));
129+
if (cnt < 0) {
130+
err = cnt;
131+
goto out;
132+
}
133+
label_pc = bsearch(&pc, labels.pcs, labels.cnt, sizeof(*labels.pcs), cmp_u32);
134+
label = "";
135+
colon = "";
136+
if (label_pc) {
137+
label = labels.names[label_pc - labels.pcs];
138+
colon = ":";
139+
}
140+
fprintf(text_out, "%x:\t", pc);
141+
for (i = 0; i < cnt; ++i)
142+
fprintf(text_out, "%02x ", image[pc + i]);
143+
for (i = cnt * 3; i < 12 * 3; ++i)
144+
fputc(' ', text_out);
145+
fprintf(text_out, "%s%s%s\n", label, colon, buf);
146+
pc += cnt;
147+
}
148+
149+
out:
150+
if (triple)
151+
LLVMDisposeMessage(triple);
152+
if (ctx)
153+
LLVMDisasmDispose(ctx);
154+
return err;
155+
}
156+
157+
int get_jited_program_text(int fd, char *text, size_t text_sz)
158+
{
159+
struct bpf_prog_info info = {};
160+
__u32 info_len = sizeof(info);
161+
__u32 jited_funcs, len, pc;
162+
__u32 *func_lens = NULL;
163+
FILE *text_out = NULL;
164+
uint8_t *image = NULL;
165+
int i, err = 0;
166+
167+
if (!llvm_initialized) {
168+
LLVMInitializeAllTargetInfos();
169+
LLVMInitializeAllTargetMCs();
170+
LLVMInitializeAllDisassemblers();
171+
llvm_initialized = 1;
172+
}
173+
174+
text_out = fmemopen(text, text_sz, "w");
175+
if (!ASSERT_OK_PTR(text_out, "open_memstream")) {
176+
err = -errno;
177+
goto out;
178+
}
179+
180+
/* first call is to find out jited program len */
181+
err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
182+
if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd #1"))
183+
goto out;
184+
185+
len = info.jited_prog_len;
186+
image = malloc(len);
187+
if (!ASSERT_OK_PTR(image, "malloc(info.jited_prog_len)")) {
188+
err = -ENOMEM;
189+
goto out;
190+
}
191+
192+
jited_funcs = info.nr_jited_func_lens;
193+
func_lens = malloc(jited_funcs * sizeof(__u32));
194+
if (!ASSERT_OK_PTR(func_lens, "malloc(info.nr_jited_func_lens)")) {
195+
err = -ENOMEM;
196+
goto out;
197+
}
198+
199+
memset(&info, 0, sizeof(info));
200+
info.jited_prog_insns = (__u64)image;
201+
info.jited_prog_len = len;
202+
info.jited_func_lens = (__u64)func_lens;
203+
info.nr_jited_func_lens = jited_funcs;
204+
err = bpf_prog_get_info_by_fd(fd, &info, &info_len);
205+
if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd #2"))
206+
goto out;
207+
208+
for (pc = 0, i = 0; i < jited_funcs; ++i) {
209+
fprintf(text_out, "func #%d:\n", i);
210+
disasm_one_func(text_out, image + pc, func_lens[i]);
211+
fprintf(text_out, "\n");
212+
pc += func_lens[i];
213+
}
214+
215+
out:
216+
if (text_out)
217+
fclose(text_out);
218+
if (image)
219+
free(image);
220+
if (func_lens)
221+
free(func_lens);
222+
return err;
223+
}
224+
225+
#else /* HAVE_LLVM_SUPPORT */
226+
227+
int get_jited_program_text(int fd, char *text, size_t text_sz)
228+
{
229+
if (env.verbosity >= VERBOSE_VERY)
230+
printf("compiled w/o llvm development libraries, can't dis-assembly binary code");
231+
return -EOPNOTSUPP;
232+
}
233+
234+
#endif /* HAVE_LLVM_SUPPORT */
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2+
3+
#ifndef __JIT_DISASM_HELPERS_H
4+
#define __JIT_DISASM_HELPERS_H
5+
6+
#include <stddef.h>
7+
8+
int get_jited_program_text(int fd, char *text, size_t text_sz);
9+
10+
#endif /* __JIT_DISASM_HELPERS_H */

0 commit comments

Comments
 (0)