Skip to content

Commit 8c90053

Browse files
Jiong Wangborkmann
authored andcommitted
nfp: bpf: implement memory bulk copy for length bigger than 32-bytes
When the gathered copy length is bigger than 32-bytes and within 128-bytes (the maximum length a single CPP Pull/Push request can finish), the strategy of read/write are changeed into: * Read. - use direct reference mode when length is within 32-bytes. - use indirect mode when length is bigger than 32-bytes. * Write. - length <= 8-bytes use write8 (direct_ref). - length <= 32-byte and 4-bytes aligned use write32 (direct_ref). - length <= 32-bytes but not 4-bytes aligned use write8 (indirect_ref). - length > 32-bytes and 4-bytes aligned use write32 (indirect_ref). - length > 32-bytes and not 4-bytes aligned and <= 40-bytes use write32 (direct_ref) to finish the first 32-bytes. use write8 (direct_ref) to finish all remaining hanging part. - length > 32-bytes and not 4-bytes aligned use write32 (indirect_ref) to finish those 4-byte aligned parts. use write8 (direct_ref) to finish all remaining hanging part. Signed-off-by: Jiong Wang <[email protected]> Reviewed-by: Jakub Kicinski <[email protected]> Signed-off-by: Daniel Borkmann <[email protected]>
1 parent 9879a38 commit 8c90053

File tree

1 file changed

+45
-7
lines changed
  • drivers/net/ethernet/netronome/nfp/bpf

1 file changed

+45
-7
lines changed

drivers/net/ethernet/netronome/nfp/bpf/jit.c

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -544,16 +544,18 @@ static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
544544
unsigned int i;
545545
u8 xfer_num;
546546

547-
if (WARN_ON_ONCE(len > 32))
548-
return -EOPNOTSUPP;
549-
550547
off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
551548
src_base = reg_a(meta->insn.src_reg * 2);
552549
xfer_num = round_up(len, 4) / 4;
553550

551+
/* Setup PREV_ALU fields to override memory read length. */
552+
if (len > 32)
553+
wrp_immed(nfp_prog, reg_none(),
554+
CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
555+
554556
/* Memory read from source addr into transfer-in registers. */
555-
emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base, off,
556-
xfer_num - 1, true);
557+
emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base,
558+
off, xfer_num - 1, true, len > 32);
557559

558560
/* Move from transfer-in to transfer-out. */
559561
for (i = 0; i < xfer_num; i++)
@@ -566,18 +568,54 @@ static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
566568
emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
567569
reg_a(meta->paired_st->dst_reg * 2), off, len - 1,
568570
true);
569-
} else if (IS_ALIGNED(len, 4)) {
571+
} else if (len <= 32 && IS_ALIGNED(len, 4)) {
570572
/* Use single direct_ref write32. */
571573
emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
572574
reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1,
573575
true);
574-
} else {
576+
} else if (len <= 32) {
575577
/* Use single indirect_ref write8. */
576578
wrp_immed(nfp_prog, reg_none(),
577579
CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, len - 1));
578580
emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
579581
reg_a(meta->paired_st->dst_reg * 2), off,
580582
len - 1, true);
583+
} else if (IS_ALIGNED(len, 4)) {
584+
/* Use single indirect_ref write32. */
585+
wrp_immed(nfp_prog, reg_none(),
586+
CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
587+
emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
588+
reg_a(meta->paired_st->dst_reg * 2), off,
589+
xfer_num - 1, true);
590+
} else if (len <= 40) {
591+
/* Use one direct_ref write32 to write the first 32-bytes, then
592+
* another direct_ref write8 to write the remaining bytes.
593+
*/
594+
emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
595+
reg_a(meta->paired_st->dst_reg * 2), off, 7,
596+
true);
597+
598+
off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32,
599+
imm_b(nfp_prog));
600+
emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8,
601+
reg_a(meta->paired_st->dst_reg * 2), off, len - 33,
602+
true);
603+
} else {
604+
/* Use one indirect_ref write32 to write 4-bytes aligned length,
605+
* then another direct_ref write8 to write the remaining bytes.
606+
*/
607+
u8 new_off;
608+
609+
wrp_immed(nfp_prog, reg_none(),
610+
CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 2));
611+
emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
612+
reg_a(meta->paired_st->dst_reg * 2), off,
613+
xfer_num - 2, true);
614+
new_off = meta->paired_st->off + (xfer_num - 1) * 4;
615+
off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog));
616+
emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b,
617+
xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off,
618+
(len & 0x3) - 1, true);
581619
}
582620

583621
/* TODO: The following extra load is to make sure data flow be identical

0 commit comments

Comments
 (0)