rds: Introduce heartbeat interval

RDS' heartbeat mechanism polls every second for a heartbeat pong. When
received, it immediately sends another heartbeat ping.

In systems with many connections, this may be way to aggressive. Hence
introduce the rds_conn_hb_interval module parameter and a simple state
to the heartbeat mechanism, in order to delay sending the ping after
the heartbeat pong has been received.

Also, pseudo randomize the delay from 50% to 150% of the interval, to
avoid many heartbeat pings to be sent in close proximity in time.

Orabug: 30418039

Signed-off-by: Håkon Bugge <[email protected]>
Reviewed-by: Ka-Cheong Poon <[email protected]>
Tested-by: Michael Nowak <[email protected]>

---

v1->v2:
	* Added Ka-Cheong's r-b
	* Added Mike's t-b

Signed-off-by: Somasundaram Krishnasamy <[email protected]>

Author: Hakon-Bugge

net/rds/rds.h

@@ -251,6 +251,11 @@ enum rds_conn_drop_src {
 	DR_TCP_SEND_FAIL,
 };
 
+enum rds_hb_state {
+	HB_PING_SENT,
+	HB_PONG_RCVD,
+};
+
 #define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
 
 /* Per mpath connection state */
@@ -311,6 +316,7 @@ struct rds_conn_path {
 	unsigned int		cp_pending_flush;
 
 	unsigned long           cp_hb_start;
+	enum rds_hb_state	cp_hb_state;
 
 	unsigned int		cp_rdsinfo_pending;
 

net/rds/threads.c

@@ -38,7 +38,10 @@
 
 static unsigned int rds_conn_hb_timeout = 0;
 module_param(rds_conn_hb_timeout, int, 0444);
-MODULE_PARM_DESC(rds_conn_hb_timeout, " Connection heartbeat timeout");
+MODULE_PARM_DESC(rds_conn_hb_timeout, " Connection heartbeat timeout (seconds)");
+static unsigned int rds_conn_hb_interval = 10;
+module_param(rds_conn_hb_interval, int, 0444);
+MODULE_PARM_DESC(rds_conn_hb_interval, " Connection heartbeat interval (seconds)");
 
 
 /*
@@ -129,9 +132,10 @@ void rds_connect_path_complete(struct rds_conn_path *cp, int curr)
 	rds_cond_queue_send_work(cp, 0);
 	rds_clear_queued_recv_work_bit(cp);
 	rds_cond_queue_recv_work(cp, 0);
+	cp->cp_hb_start = 0;
+	cp->cp_hb_state = HB_PONG_RCVD;
 	queue_delayed_work(cp->cp_wq, &cp->cp_hb_w, 0);
 	cancel_delayed_work(&cp->cp_reconn_w);
-	cp->cp_hb_start = 0;
 
 	rds_update_avg_connect_time(cp);
 	cp->cp_connection_start = get_seconds();
@@ -336,6 +340,7 @@ void rds_hb_worker(struct work_struct *work)
 						struct rds_conn_path,
 						cp_hb_w.work);
 	unsigned long now = get_seconds();
+	unsigned long delay = HZ;
 	int ret;
 	struct rds_connection *conn = cp->cp_conn;
 
@@ -344,24 +349,38 @@ void rds_hb_worker(struct work_struct *work)
 		return;
 
 	if (rds_conn_path_state(cp) == RDS_CONN_UP) {
-		if (!cp->cp_hb_start) {
+		switch (cp->cp_hb_state) {
+		case HB_PING_SENT:
+			if (!cp->cp_hb_start) {
+				cp->cp_hb_state = HB_PONG_RCVD;
+				/* Pseudo random from 50% to 150% of interval */
+				delay = msecs_to_jiffies(rds_conn_hb_interval * 1000 / 2) +
+					msecs_to_jiffies(prandom_u32() % rds_conn_hb_interval * 1000);
+			} else if (now - cp->cp_hb_start > rds_conn_hb_timeout) {
+				rds_rtd_ptr(RDS_RTD_CM,
+					    "RDS/IB: connection <%pI6c,%pI6c,%d> timed out (0x%lx,0x%lx)..discon and recon\n",
+					    &conn->c_laddr, &conn->c_faddr,
+					    conn->c_tos, cp->cp_hb_start, now);
+				rds_conn_path_drop(cp, DR_HB_TIMEOUT);
+				return;
+			}
+			break;
+
+		case HB_PONG_RCVD:
 			ret = rds_send_hb(cp->cp_conn, 0);
+
 			if (ret) {
 				rds_rtd(RDS_RTD_ERR_EXT,
 					"RDS/IB: rds_hb_worker: failed %d\n",
 					ret);
 				return;
 			}
 			cp->cp_hb_start = now;
-		} else if (now - cp->cp_hb_start > rds_conn_hb_timeout) {
-			rds_rtd_ptr(RDS_RTD_CM,
-				    "RDS/IB: connection <%pI6c,%pI6c,%d> timed out (0x%lx,0x%lx)..discon and recon\n",
-				    &conn->c_laddr, &conn->c_faddr,
-				    conn->c_tos, cp->cp_hb_start, now);
-			rds_conn_path_drop(cp, DR_HB_TIMEOUT);
-			return;
+			cp->cp_hb_state = HB_PING_SENT;
+			break;
 		}
-		queue_delayed_work(cp->cp_wq, &cp->cp_hb_w, HZ);
+
+		queue_delayed_work(cp->cp_wq, &cp->cp_hb_w, delay);
 	}
 }