Skip to content

Commit 815ce86

Browse files
juamagarBrian Maly
juamagar
authored and
Brian Maly
committed
rds/rdma: Send info to userspace, even if connnection is down.
Currently, kernel sends most info to userspace only when connection is in UP state. With this change, kernel can send info even if connection is not UP along with connection state. This info would help in seeing historical value, which helps in debugging. RDS IB Connections: LocalAddr ... DstQPNo State 192.168.100.118 ... 462 UP Orabug: 36529561 Signed-off-by: Juan Garcia <[email protected]> Reviewed-by: Arumugam Kolappan <[email protected]> Signed-off-by: Brian Maly <[email protected]>
1 parent c834e79 commit 815ce86

File tree

2 files changed

+93
-79
lines changed

2 files changed

+93
-79
lines changed

‎include/uapi/linux/rds.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ struct rds_info_rdma_connection {
302302
__u32 rdma_mr_size;
303303
__u8 tos;
304304
__u8 sl;
305+
__u8 conn_state;
305306
__u32 cache_allocs;
306307
__u32 frag;
307308
__u16 flow_ctl_post_credit;
@@ -339,6 +340,7 @@ struct rds6_info_rdma_connection {
339340
__u32 rdma_mr_size;
340341
__u8 tos;
341342
__u8 sl;
343+
__u8 conn_state;
342344
__u32 cache_allocs;
343345
__u32 frag;
344346
__u16 flow_ctl_post_credit;
@@ -505,4 +507,13 @@ struct rds_rdma_send_notify {
505507
#define RDS_RDMA_REMOTE_COMPLETE 0x0080 /* Notify when data is available */
506508
#define RDS_SEND_NOTIFY_ME 0x0100 /* Notify when operation completes */
507509

510+
511+
enum {
512+
CONN_STATE_DOWN = 0,
513+
CONN_STATE_CONNECTING,
514+
CONN_STATE_DISCONNECTING,
515+
CONN_STATE_UP,
516+
CONN_STATE_RESETTING,
517+
CONN_STATE_ERROR,
518+
};
508519
#endif /* IB_RDS_H */

‎net/rds/ib.c

Lines changed: 82 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,7 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
922922
memset(iinfo, 0, sizeof(*iinfo));
923923

924924
/* We will only ever look at IB transports */
925-
if (conn->c_trans != &rds_ib_transport)
925+
if (conn->c_trans != &rds_ib_transport || !ic)
926926
return 0;
927927
if (conn->c_isv6)
928928
return 0;
@@ -933,56 +933,57 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
933933
memset(&iinfo->src_gid, 0, sizeof(iinfo->src_gid));
934934
memset(&iinfo->dst_gid, 0, sizeof(iinfo->dst_gid));
935935

936-
iinfo->qp_num = -1;
937-
iinfo->dst_qp_num = -1;
938-
939-
if (ic) {
940-
iinfo->tos = conn->c_tos;
941-
iinfo->sl = ic->i_sl;
942-
iinfo->frag = ic->i_frag_sz;
943-
}
944-
945936
if (rds_conn_state(conn) == RDS_CONN_UP) {
946937
struct rds_ib_device *rds_ibdev;
947938

948939
rdma_read_gids(ic->i_cm_id, (union ib_gid *)&iinfo->src_gid,
949940
(union ib_gid *)&iinfo->dst_gid);
950-
951941
rds_ibdev = ic->rds_ibdev;
952-
iinfo->max_send_wr = ic->i_send_ring.w_nr;
953-
iinfo->max_recv_wr = ic->i_recv_ring.w_nr;
954942
iinfo->max_send_sge = rds_ibdev->max_sge;
955-
iinfo->qp_num = ic->i_qp_num;
956-
iinfo->dst_qp_num = ic->i_dst_qp_num;
957-
iinfo->recv_alloc_ctr = (uint32_t)atomic64_read(&ic->i_recv_ring.w_alloc_ctr);
958-
iinfo->recv_free_ctr = (uint32_t)atomic64_read(&ic->i_recv_ring.w_free_ctr);
959-
iinfo->flow_ctl_post_credit =
960-
IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));
961-
iinfo->flow_ctl_send_credit =
962-
IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits));
963943
rds_ib_get_mr_info(rds_ibdev, iinfo);
964-
iinfo->cache_allocs = atomic_read(&ic->i_cache_allocs);
965-
iinfo->send_alloc_ctr = (uint32_t)atomic64_read(&ic->i_send_ring.w_alloc_ctr);
966-
iinfo->send_free_ctr = (uint32_t)atomic64_read(&ic->i_send_ring.w_free_ctr);
967-
iinfo->send_bytes =
968-
(uint64_t)atomic64_read(&conn->c_send_bytes);
969-
iinfo->recv_bytes =
970-
(uint64_t)atomic64_read(&conn->c_recv_bytes);
971-
iinfo->r_read_bytes =
972-
(uint64_t)atomic64_read(&ic->i_r_read_bytes);
973-
iinfo->r_write_bytes =
974-
(uint64_t)atomic64_read(&ic->i_r_write_bytes);
975-
iinfo->tx_poll_ts = jiffies_to_msecs(now - ic->i_tx_poll_ts);
976-
iinfo->rx_poll_ts = jiffies_to_msecs(now - ic->i_rx_poll_ts);
977-
iinfo->tx_poll_cnt =
978-
(uint64_t)atomic64_read(&ic->i_tx_poll_cnt);
979-
iinfo->rx_poll_cnt =
980-
(uint64_t)atomic64_read(&ic->i_rx_poll_cnt);
981-
iinfo->scq_vector = ic->i_scq_vector;
982-
iinfo->rcq_vector = ic->i_rcq_vector;
944+
983945
iinfo->scq_irq = ib_get_vector_irqn(rds_ibdev->dev, ic->i_scq_vector);
984946
iinfo->rcq_irq = ib_get_vector_irqn(rds_ibdev->dev, ic->i_rcq_vector);
985947
}
948+
949+
iinfo->tos = conn->c_tos;
950+
iinfo->sl = ic->i_sl;
951+
iinfo->conn_state = rds_conn_state(conn);
952+
iinfo->frag = ic->i_frag_sz;
953+
iinfo->max_send_wr = ic->i_send_ring.w_nr;
954+
iinfo->max_recv_wr = ic->i_recv_ring.w_nr;
955+
iinfo->qp_num = ic->i_qp_num;
956+
iinfo->dst_qp_num = ic->i_dst_qp_num;
957+
iinfo->recv_alloc_ctr =
958+
(uint32_t)atomic64_read(&ic->i_recv_ring.w_alloc_ctr);
959+
iinfo->recv_free_ctr =
960+
(uint32_t)atomic64_read(&ic->i_recv_ring.w_free_ctr);
961+
iinfo->flow_ctl_post_credit =
962+
IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));
963+
iinfo->flow_ctl_send_credit =
964+
IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits));
965+
iinfo->cache_allocs = atomic_read(&ic->i_cache_allocs);
966+
iinfo->send_alloc_ctr =
967+
(uint32_t)atomic64_read(&ic->i_send_ring.w_alloc_ctr);
968+
iinfo->send_free_ctr =
969+
(uint32_t)atomic64_read(&ic->i_send_ring.w_free_ctr);
970+
iinfo->send_bytes =
971+
(uint64_t)atomic64_read(&conn->c_send_bytes);
972+
iinfo->recv_bytes =
973+
(uint64_t)atomic64_read(&conn->c_recv_bytes);
974+
iinfo->r_read_bytes =
975+
(uint64_t)atomic64_read(&ic->i_r_read_bytes);
976+
iinfo->r_write_bytes =
977+
(uint64_t)atomic64_read(&ic->i_r_write_bytes);
978+
iinfo->tx_poll_ts = jiffies_to_msecs(now - ic->i_tx_poll_ts);
979+
iinfo->rx_poll_ts = jiffies_to_msecs(now - ic->i_rx_poll_ts);
980+
iinfo->tx_poll_cnt =
981+
(uint64_t)atomic64_read(&ic->i_tx_poll_cnt);
982+
iinfo->rx_poll_cnt =
983+
(uint64_t)atomic64_read(&ic->i_rx_poll_cnt);
984+
iinfo->scq_vector = ic->i_scq_vector;
985+
iinfo->rcq_vector = ic->i_rcq_vector;
986+
986987
return 1;
987988
}
988989

@@ -998,7 +999,7 @@ static int rds6_ib_conn_info_visitor(struct rds_connection *conn,
998999
memset(iinfo6, 0, sizeof(*iinfo6));
9991000

10001001
/* We will only ever look at IB transports */
1001-
if (conn->c_trans != &rds_ib_transport)
1002+
if (conn->c_trans != &rds_ib_transport || !ic)
10021003
return 0;
10031004

10041005
iinfo6->src_addr = conn->c_laddr;
@@ -1007,56 +1008,58 @@ static int rds6_ib_conn_info_visitor(struct rds_connection *conn,
10071008
memset(&iinfo6->src_gid, 0, sizeof(iinfo6->src_gid));
10081009
memset(&iinfo6->dst_gid, 0, sizeof(iinfo6->dst_gid));
10091010

1010-
iinfo6->qp_num = -1;
1011-
iinfo6->dst_qp_num = -1;
1012-
1013-
if (ic) {
1014-
iinfo6->tos = conn->c_tos;
1015-
iinfo6->sl = ic->i_sl;
1016-
iinfo6->frag = ic->i_frag_sz;
1017-
}
1018-
10191011
if (rds_conn_state(conn) == RDS_CONN_UP) {
10201012
struct rds_ib_device *rds_ibdev;
10211013

10221014
rdma_read_gids(ic->i_cm_id, (union ib_gid *)&iinfo6->src_gid,
10231015
(union ib_gid *)&iinfo6->dst_gid);
10241016

10251017
rds_ibdev = ic->rds_ibdev;
1026-
iinfo6->max_send_wr = ic->i_send_ring.w_nr;
1027-
iinfo6->max_recv_wr = ic->i_recv_ring.w_nr;
10281018
iinfo6->max_send_sge = rds_ibdev->max_sge;
1029-
iinfo6->qp_num = ic->i_qp_num;
1030-
iinfo6->dst_qp_num = ic->i_dst_qp_num;
1031-
iinfo6->recv_alloc_ctr = (uint32_t)atomic64_read(&ic->i_recv_ring.w_alloc_ctr);
1032-
iinfo6->recv_free_ctr = (uint32_t)atomic64_read(&ic->i_recv_ring.w_free_ctr);
1033-
iinfo6->flow_ctl_post_credit =
1034-
IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));
1035-
iinfo6->flow_ctl_send_credit =
1036-
IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits));
10371019
rds6_ib_get_mr_info(rds_ibdev, iinfo6);
1038-
iinfo6->cache_allocs = atomic_read(&ic->i_cache_allocs);
1039-
iinfo6->send_alloc_ctr = (uint32_t)atomic64_read(&ic->i_send_ring.w_alloc_ctr);
1040-
iinfo6->send_free_ctr = (uint32_t)atomic64_read(&ic->i_send_ring.w_free_ctr);
1041-
iinfo6->send_bytes =
1042-
(uint64_t)atomic64_read(&conn->c_send_bytes);
1043-
iinfo6->recv_bytes =
1044-
(uint64_t)atomic64_read(&conn->c_recv_bytes);
1045-
iinfo6->r_read_bytes =
1046-
(uint64_t)atomic64_read(&ic->i_r_read_bytes);
1047-
iinfo6->r_write_bytes =
1048-
(uint64_t)atomic64_read(&ic->i_r_write_bytes);
1049-
iinfo6->tx_poll_ts = jiffies_to_msecs(now - ic->i_tx_poll_ts);
1050-
iinfo6->rx_poll_ts = jiffies_to_msecs(now - ic->i_rx_poll_ts);
1051-
iinfo6->tx_poll_cnt =
1052-
(uint64_t)atomic64_read(&ic->i_tx_poll_cnt);
1053-
iinfo6->rx_poll_cnt =
1054-
(uint64_t)atomic64_read(&ic->i_rx_poll_cnt);
1055-
iinfo6->scq_vector = ic->i_scq_vector;
1056-
iinfo6->rcq_vector = ic->i_rcq_vector;
1020+
10571021
iinfo6->scq_irq = ib_get_vector_irqn(rds_ibdev->dev, ic->i_scq_vector);
10581022
iinfo6->rcq_irq = ib_get_vector_irqn(rds_ibdev->dev, ic->i_rcq_vector);
10591023
}
1024+
1025+
iinfo6->tos = conn->c_tos;
1026+
iinfo6->sl = ic->i_sl;
1027+
iinfo6->conn_state = rds_conn_state(conn);
1028+
iinfo6->frag = ic->i_frag_sz;
1029+
iinfo6->max_send_wr = ic->i_send_ring.w_nr;
1030+
iinfo6->max_recv_wr = ic->i_recv_ring.w_nr;
1031+
iinfo6->qp_num = ic->i_qp_num;
1032+
iinfo6->dst_qp_num = ic->i_dst_qp_num;
1033+
iinfo6->recv_alloc_ctr =
1034+
(uint32_t)atomic64_read(&ic->i_recv_ring.w_alloc_ctr);
1035+
iinfo6->recv_free_ctr =
1036+
(uint32_t)atomic64_read(&ic->i_recv_ring.w_free_ctr);
1037+
iinfo6->flow_ctl_post_credit =
1038+
IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));
1039+
iinfo6->flow_ctl_send_credit =
1040+
IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits));
1041+
iinfo6->cache_allocs = atomic_read(&ic->i_cache_allocs);
1042+
iinfo6->send_alloc_ctr =
1043+
(uint32_t)atomic64_read(&ic->i_send_ring.w_alloc_ctr);
1044+
iinfo6->send_free_ctr =
1045+
(uint32_t)atomic64_read(&ic->i_send_ring.w_free_ctr);
1046+
iinfo6->send_bytes =
1047+
(uint64_t)atomic64_read(&conn->c_send_bytes);
1048+
iinfo6->recv_bytes =
1049+
(uint64_t)atomic64_read(&conn->c_recv_bytes);
1050+
iinfo6->r_read_bytes =
1051+
(uint64_t)atomic64_read(&ic->i_r_read_bytes);
1052+
iinfo6->r_write_bytes =
1053+
(uint64_t)atomic64_read(&ic->i_r_write_bytes);
1054+
iinfo6->tx_poll_ts = jiffies_to_msecs(now - ic->i_tx_poll_ts);
1055+
iinfo6->rx_poll_ts = jiffies_to_msecs(now - ic->i_rx_poll_ts);
1056+
iinfo6->tx_poll_cnt =
1057+
(uint64_t)atomic64_read(&ic->i_tx_poll_cnt);
1058+
iinfo6->rx_poll_cnt =
1059+
(uint64_t)atomic64_read(&ic->i_rx_poll_cnt);
1060+
iinfo6->scq_vector = ic->i_scq_vector;
1061+
iinfo6->rcq_vector = ic->i_rcq_vector;
1062+
10601063
return 1;
10611064
}
10621065
#endif

0 commit comments

Comments
 (0)