rds: ib: need to flush neighbor cache for local peer connections on failover

On RDS connections to other hosts, the neighbor cache on the peer side
normally will get flushed by means of GARP packets. For local connections
(connections on the same host), GARP packets does not cause neighbor cache
to be updated. To prevent RDS to use bad entries in the neighbor cache we
need to flush the neighbor cache entries TO the IP-address when the
IP-address is moved due to an ADDR_CHANGED event.

Orabug: 30417190

Signed-off-by: Dag Moxnes <[email protected]>
Reviewed-by: Ka-Cheong Poon <[email protected]>
Signed-off-by: Somasundaram Krishnasamy <[email protected]>

Author: Dag Moxnes

net/rds/ib.c

@@ -851,14 +851,8 @@ static unsigned int neigh_flush_interval = 750;
 /* Should be large enough to hold the flush message. */
 static unsigned int flush_buf_len = 48;
 
-/* Given an rds_connection, flush the peer address' neighbor cache entry.
- * If the peer is not in the same network as us, nothing will be flushed.
- *
- * @net: the connection's namespace
- * @conn: pointer to the connection
- */
-void rds_ib_flush_neigh(struct net *net,
-			struct rds_connection *conn)
+static void __flush_neigh_conn(struct net *net,
+			       struct rds_connection *conn)
 {
 	struct sockaddr_nl nlsa = { .nl_family = AF_NETLINK };
 	u64 timenow = jiffies_to_msecs(get_jiffies_64());
@@ -954,4 +948,30 @@ void rds_ib_flush_neigh(struct net *net,
 		kfree(sndbuf);
 }
 
+/* Given an rds_connection, flush the peer address' neighbor cache entry.
+ * If the peer is not in the same network as us, nothing will be flushed.
+ *
+ * @net: the connection's namespace
+ * @conn: pointer to the connection
+ * @flush_local_peer: Flush neighbor for the peer, if it is local, instead
+ * of conn
+ */
+void rds_ib_flush_neigh(struct net *net,
+			struct rds_connection *conn, bool flush_local_peer)
+{
+	if (flush_local_peer && conn->c_loopback) {
+		struct rds_connection *peer;
+
+		/* Note the swapped d/saddr */
+		peer = rds_conn_find(rds_conn_net(conn),
+				     &conn->c_faddr, &conn->c_laddr,
+				     conn->c_trans, conn->c_tos,
+				     0);
+		if (peer)
+			__flush_neigh_conn(net, peer);
+	} else {
+		__flush_neigh_conn(net, conn);
+	}
+}
+
 MODULE_LICENSE("GPL");

net/rds/ib.h

@@ -552,7 +552,9 @@ extern struct workqueue_struct *rds_aux_wq;
 extern struct rds_transport rds_ib_transport;
 extern void rds_ib_add_one(struct ib_device *device);
 extern void rds_ib_remove_one(struct ib_device *device, void *client_data);
-extern void rds_ib_flush_neigh(struct net *net, struct rds_connection *conn);
+extern void rds_ib_flush_neigh(struct net *net,
+			       struct rds_connection *conn,
+			       bool flush_local_peer);
 void rds_ib_srq_exit(struct rds_ib_device *rds_ibdev);
 int rds_ib_srq_init(struct rds_ib_device *rds_ibdev);
 

net/rds/rdma_transport.c

@@ -95,6 +95,15 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 	struct rds_connection *conn;
 	struct rds_transport *trans = &rds_ib_transport;
 	int ret = 0;
+	/* ADDR_CHANGE event indicates that the local address has moved
+	 * to a different device, most likely due to failover/failback.
+	 * If this is a local connection (a connection to this host), we need
+	 * to flush the neighbor cache entry for the peer side of the
+	 * connection. In this case we do not need to flush this side of the
+	 * connection. If this is not a local connection, we still flush
+	 * the neighbor cache for the local side of the connection.
+	 */
+	bool flush_local_peer = event->event == RDMA_CM_EVENT_ADDR_CHANGE;
 	int *err;
 
 	conn = rds_ib_get_conn(cm_id);
@@ -136,7 +145,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 			/* These events might indicate the IP being moved,
 			 * hence flush the address
 			 */
-			rds_ib_flush_neigh(&init_net, conn);
+			rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 		rds_rtd(RDS_RTD_CM, "Bailing, conn %p being shut down, ret: %d\n",
 			conn, ret);
 		goto out;
@@ -212,7 +221,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 
 	case RDMA_CM_EVENT_ROUTE_ERROR:
 		/* IP might have been moved so flush the ARP entry and retry */
-		rds_ib_flush_neigh(&init_net, conn);
+		rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 
 		rds_rtd_ptr(RDS_RTD_ERR,
 			    "ROUTE_ERROR: conn %p, calling rds_conn_drop <%pI6c,%pI6c,%d>\n",
@@ -228,7 +237,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 
 	case RDMA_CM_EVENT_ADDR_ERROR:
 		/* IP might have been moved so flush the ARP entry and retry */
-		rds_ib_flush_neigh(&init_net, conn);
+		rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 
 		rds_rtd_ptr(RDS_RTD_ERR,
 			    "ADDR_ERROR: conn %p, calling rds_conn_drop <%pI6c,%pI6c,%d>\n",
@@ -242,7 +251,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 	case RDMA_CM_EVENT_UNREACHABLE:
 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
 		/* IP might have been moved so flush the ARP entry and retry */
-		rds_ib_flush_neigh(&init_net, conn);
+		rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 
 		rds_rtd_ptr(RDS_RTD_ERR,
 			    "CONN/UNREACHABLE/RMVAL ERR: conn %p, calling rds_conn_drop <%pI6c,%pI6c,%d>\n",
@@ -254,7 +263,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 
 	case RDMA_CM_EVENT_REJECTED:
 		/* May be due to ARP cache containing an incorrect dmac, hence flush it */
-		rds_ib_flush_neigh(&init_net, conn);
+		rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 
 		err = (int *)event->param.conn.private_data;
 
@@ -300,7 +309,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 
 	case RDMA_CM_EVENT_ADDR_CHANGE:
 		/* IP might have been moved so flush the ARP entry and retry */
-		rds_ib_flush_neigh(&init_net, conn);
+		rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 
 		rds_rtd_ptr(RDS_RTD_CM_EXT,
 			    "ADDR_CHANGE event <%pI6c,%pI6c>\n",
@@ -315,7 +324,7 @@ static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id,
 
 	case RDMA_CM_EVENT_DISCONNECTED:
 		/* IP might have been moved so flush the ARP entry and retry */
-		rds_ib_flush_neigh(&init_net, conn);
+		rds_ib_flush_neigh(&init_net, conn, flush_local_peer);
 
 		rds_rtd_ptr(RDS_RTD_CM,
 			    "DISCONNECT event - dropping conn %p <%pI6c,%pI6c,%d>\n",