@@ -3991,7 +3991,7 @@ static bool idset_contains(struct bpf_idset *s, u32 id)
3991
3991
u32 i;
3992
3992
3993
3993
for (i = 0; i < s->count; ++i)
3994
- if (s->ids[i] == id )
3994
+ if (s->ids[i] == (id & ~BPF_ADD_CONST) )
3995
3995
return true;
3996
3996
3997
3997
return false;
@@ -4001,7 +4001,7 @@ static int idset_push(struct bpf_idset *s, u32 id)
4001
4001
{
4002
4002
if (WARN_ON_ONCE(s->count >= ARRAY_SIZE(s->ids)))
4003
4003
return -EFAULT;
4004
- s->ids[s->count++] = id;
4004
+ s->ids[s->count++] = id & ~BPF_ADD_CONST ;
4005
4005
return 0;
4006
4006
}
4007
4007
@@ -4438,8 +4438,20 @@ static bool __is_pointer_value(bool allow_ptr_leaks,
4438
4438
static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
4439
4439
struct bpf_reg_state *src_reg)
4440
4440
{
4441
- if (src_reg->type == SCALAR_VALUE && !src_reg->id &&
4442
- !tnum_is_const(src_reg->var_off))
4441
+ if (src_reg->type != SCALAR_VALUE)
4442
+ return;
4443
+
4444
+ if (src_reg->id & BPF_ADD_CONST) {
4445
+ /*
4446
+ * The verifier is processing rX = rY insn and
4447
+ * rY->id has special linked register already.
4448
+ * Cleared it, since multiple rX += const are not supported.
4449
+ */
4450
+ src_reg->id = 0;
4451
+ src_reg->off = 0;
4452
+ }
4453
+
4454
+ if (!src_reg->id && !tnum_is_const(src_reg->var_off))
4443
4455
/* Ensure that src_reg has a valid ID that will be copied to
4444
4456
* dst_reg and then will be used by find_equal_scalars() to
4445
4457
* propagate min/max range.
@@ -14042,6 +14054,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
14042
14054
struct bpf_func_state *state = vstate->frame[vstate->curframe];
14043
14055
struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
14044
14056
struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
14057
+ bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
14045
14058
u8 opcode = BPF_OP(insn->code);
14046
14059
int err;
14047
14060
@@ -14064,11 +14077,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
14064
14077
14065
14078
if (dst_reg->type != SCALAR_VALUE)
14066
14079
ptr_reg = dst_reg;
14067
- else
14068
- /* Make sure ID is cleared otherwise dst_reg min/max could be
14069
- * incorrectly propagated into other registers by find_equal_scalars()
14070
- */
14071
- dst_reg->id = 0;
14080
+
14072
14081
if (BPF_SRC(insn->code) == BPF_X) {
14073
14082
src_reg = ®s[insn->src_reg];
14074
14083
if (src_reg->type != SCALAR_VALUE) {
@@ -14132,7 +14141,43 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
14132
14141
verbose(env, "verifier internal error: no src_reg\n");
14133
14142
return -EINVAL;
14134
14143
}
14135
- return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
14144
+ err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
14145
+ if (err)
14146
+ return err;
14147
+ /*
14148
+ * Compilers can generate the code
14149
+ * r1 = r2
14150
+ * r1 += 0x1
14151
+ * if r2 < 1000 goto ...
14152
+ * use r1 in memory access
14153
+ * So remember constant delta between r2 and r1 and update r1 after
14154
+ * 'if' condition.
14155
+ */
14156
+ if (env->bpf_capable && BPF_OP(insn->code) == BPF_ADD &&
14157
+ dst_reg->id && is_reg_const(src_reg, alu32)) {
14158
+ u64 val = reg_const_value(src_reg, alu32);
14159
+
14160
+ if ((dst_reg->id & BPF_ADD_CONST) ||
14161
+ /* prevent overflow in find_equal_scalars() later */
14162
+ val > (u32)S32_MAX) {
14163
+ /*
14164
+ * If the register already went through rX += val
14165
+ * we cannot accumulate another val into rx->off.
14166
+ */
14167
+ dst_reg->off = 0;
14168
+ dst_reg->id = 0;
14169
+ } else {
14170
+ dst_reg->id |= BPF_ADD_CONST;
14171
+ dst_reg->off = val;
14172
+ }
14173
+ } else {
14174
+ /*
14175
+ * Make sure ID is cleared otherwise dst_reg min/max could be
14176
+ * incorrectly propagated into other registers by find_equal_scalars()
14177
+ */
14178
+ dst_reg->id = 0;
14179
+ }
14180
+ return 0;
14136
14181
}
14137
14182
14138
14183
/* check validity of 32-bit and 64-bit arithmetic operations */
@@ -15104,12 +15149,36 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
15104
15149
static void find_equal_scalars(struct bpf_verifier_state *vstate,
15105
15150
struct bpf_reg_state *known_reg)
15106
15151
{
15152
+ struct bpf_reg_state fake_reg;
15107
15153
struct bpf_func_state *state;
15108
15154
struct bpf_reg_state *reg;
15109
15155
15110
15156
bpf_for_each_reg_in_vstate(vstate, state, reg, ({
15111
- if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
15157
+ if (reg->type != SCALAR_VALUE || reg == known_reg)
15158
+ continue;
15159
+ if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
15160
+ continue;
15161
+ if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) ||
15162
+ reg->off == known_reg->off) {
15112
15163
copy_register_state(reg, known_reg);
15164
+ } else {
15165
+ s32 saved_off = reg->off;
15166
+
15167
+ fake_reg.type = SCALAR_VALUE;
15168
+ __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off);
15169
+
15170
+ /* reg = known_reg; reg += delta */
15171
+ copy_register_state(reg, known_reg);
15172
+ /*
15173
+ * Must preserve off, id and add_const flag,
15174
+ * otherwise another find_equal_scalars() will be incorrect.
15175
+ */
15176
+ reg->off = saved_off;
15177
+
15178
+ scalar32_min_max_add(reg, &fake_reg);
15179
+ scalar_min_max_add(reg, &fake_reg);
15180
+ reg->var_off = tnum_add(reg->var_off, fake_reg.var_off);
15181
+ }
15113
15182
}));
15114
15183
}
15115
15184
@@ -16738,6 +16807,10 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
16738
16807
}
16739
16808
if (!rold->precise && exact == NOT_EXACT)
16740
16809
return true;
16810
+ if ((rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST))
16811
+ return false;
16812
+ if ((rold->id & BPF_ADD_CONST) && (rold->off != rcur->off))
16813
+ return false;
16741
16814
/* Why check_ids() for scalar registers?
16742
16815
*
16743
16816
* Consider the following BPF code:
0 commit comments