Skip to content

Commit d3c04a3

Browse files
Vijay Immanueldledford
authored andcommitted
IB/rxe: vary the source udp port for receive scaling
Select the source udp port number for a QP based on the source QPN. This provides a better spread of traffic across NIC RX queues for RC/UC QPs. Signed-off-by: Vijay Immanuel <[email protected]> Signed-off-by: Doug Ledford <[email protected]>
1 parent 5b394b2 commit d3c04a3

File tree

3 files changed

+15
-4
lines changed

3 files changed

+15
-4
lines changed

drivers/infiniband/sw/rxe/rxe_net.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -396,8 +396,8 @@ static int prepare4(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
396396
if (!memcmp(saddr, daddr, sizeof(*daddr)))
397397
pkt->mask |= RXE_LOOPBACK_MASK;
398398

399-
prepare_udp_hdr(skb, htons(RXE_ROCE_V2_SPORT),
400-
htons(ROCE_V2_UDP_DPORT));
399+
prepare_udp_hdr(skb, cpu_to_be16(qp->src_port),
400+
cpu_to_be16(ROCE_V2_UDP_DPORT));
401401

402402
prepare_ipv4_hdr(dst, skb, saddr->s_addr, daddr->s_addr, IPPROTO_UDP,
403403
av->grh.traffic_class, av->grh.hop_limit, df, xnet);
@@ -423,8 +423,8 @@ static int prepare6(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
423423
if (!memcmp(saddr, daddr, sizeof(*daddr)))
424424
pkt->mask |= RXE_LOOPBACK_MASK;
425425

426-
prepare_udp_hdr(skb, htons(RXE_ROCE_V2_SPORT),
427-
htons(ROCE_V2_UDP_DPORT));
426+
prepare_udp_hdr(skb, cpu_to_be16(qp->src_port),
427+
cpu_to_be16(ROCE_V2_UDP_DPORT));
428428

429429
prepare_ipv6_hdr(dst, skb, saddr, daddr, IPPROTO_UDP,
430430
av->grh.traffic_class,

drivers/infiniband/sw/rxe/rxe_qp.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,16 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
227227
return err;
228228
qp->sk->sk->sk_user_data = qp;
229229

230+
/* pick a source UDP port number for this QP based on
231+
* the source QPN. this spreads traffic for different QPs
232+
* across different NIC RX queues (while using a single
233+
* flow for a given QP to maintain packet order).
234+
* the port number must be in the Dynamic Ports range
235+
* (0xc000 - 0xffff).
236+
*/
237+
qp->src_port = RXE_ROCE_V2_SPORT +
238+
(hash_32_generic(qp_num(qp), 14) & 0x3fff);
239+
230240
qp->sq.max_wr = init->cap.max_send_wr;
231241
qp->sq.max_sge = init->cap.max_send_sge;
232242
qp->sq.max_inline = init->cap.max_inline_data;

drivers/infiniband/sw/rxe/rxe_verbs.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ struct rxe_qp {
248248

249249
struct socket *sk;
250250
u32 dst_cookie;
251+
u16 src_port;
251252

252253
struct rxe_av pri_av;
253254
struct rxe_av alt_av;

0 commit comments

Comments
 (0)