RDS: IB: Add proxy qp to support FRWR through RDS_GET_MR

MR registration requested through RDS_GET_MR socket option will not have
any connection details. So, there isn't an appropriate qp to post the
registration/invalidation requests. This patch solves that issue by
using a proxy qp.

Orabug: 25669255

Signed-off-by: Avinash Repaka <[email protected]>

Reviewed-by: Wei Lin Guay <[email protected]>

Tested-by: Gerald Gibson <[email protected]>
Tested-by: Efrain Galaviz <[email protected]>

Orabug: 27364391

(cherry picked from commit 950ef83)
cherry-pick-repo=linux-uek.git

Conflicts:
	net/rds/ib_cm.c
	net/rds/ib_rdma.c

Signed-off-by: Gerd Rausch <[email protected]>
Signed-off-by: Somasundaram Krishnasamy <[email protected]>

Author: Avinash Repaka

net/rds/ib.c

@@ -213,6 +213,11 @@ static void rds_ib_dev_free_dev(struct rds_ib_device *rds_ibdev)
 		rds_ib_destroy_mr_pool(rds_ibdev->mr_8k_pool);
 	if (rds_ibdev->mr_1m_pool)
 		rds_ib_destroy_mr_pool(rds_ibdev->mr_1m_pool);
+	if (rds_ibdev->use_fastreg) {
+		cancel_work_sync(&rds_ibdev->fastreg_reset_w);
+		down_write(&rds_ibdev->fastreg_lock);
+		rds_ib_destroy_fastreg(rds_ibdev);
+	}
 	if (rds_ibdev->mr)
 		ib_dereg_mr(rds_ibdev->mr);
 	if (rds_ibdev->pd)
@@ -287,6 +292,9 @@ void rds_ib_remove_one(struct ib_device *device, void *client_data)
 	struct rds_ib_device *rds_ibdev;
 	int i;
 
+	rds_rtd(RDS_RTD_RDMA_IB, "Removing ib_device: %p name: %s num_ports: %u\n",
+		device, device->name, device->phys_port_cnt);
+
 	rds_ibdev = ib_get_client_data(device, &rds_ib_client);
 	if (!rds_ibdev) {
 		rds_rtd(RDS_RTD_ACT_BND, "rds_ibdev is NULL, ib_device %p\n",
@@ -1987,6 +1995,9 @@ void rds_ib_add_one(struct ib_device *device)
 	struct ib_device_attr *dev_attr;
 	bool has_frwr, has_fmr;
 
+	rds_rtd(RDS_RTD_RDMA_IB, "Adding ib_device: %p name: %s num_ports: %u\n",
+		device, device->name, device->phys_port_cnt);
+
 	/* Only handle IB (no iWARP) devices */
 	if (device->node_type != RDMA_NODE_IB_CA)
 		return;
@@ -2065,6 +2076,16 @@ void rds_ib_add_one(struct ib_device *device)
 	pr_info("RDS/IB: %s will be used for ib_device: %s\n",
 		rds_ibdev->use_fastreg ? "FRWR" : "FMR", device->name);
 
+	if (rds_ibdev->use_fastreg) {
+		INIT_WORK(&rds_ibdev->fastreg_reset_w, rds_ib_reset_fastreg);
+		init_rwsem(&rds_ibdev->fastreg_lock);
+		atomic_set(&rds_ibdev->fastreg_wrs, RDS_IB_DEFAULT_FREG_WR);
+		if (rds_ib_setup_fastreg(rds_ibdev)) {
+			pr_err("RDS/IB: Failed to setup fastreg resources\n");
+			goto put_dev;
+		}
+	}
+
 	rds_ibdev->mr_1m_pool =
 		rds_ib_create_mr_pool(rds_ibdev, RDS_IB_MR_1M_POOL);
 	if (IS_ERR(rds_ibdev->mr_1m_pool)) {

net/rds/ib.h

@@ -52,6 +52,8 @@
 
 #define	RDS_IB_CLEAN_CACHE	1
 
+#define RDS_IB_DEFAULT_FREG_PORT_NUM	1
+
 extern struct rw_semaphore rds_ib_devices_lock;
 extern struct list_head rds_ib_devices;
 
@@ -422,7 +424,17 @@ struct rds_ib_device {
 	struct list_head	conn_list;
 	struct ib_device	*dev;
 	struct ib_pd		*pd;
+
 	bool			use_fastreg;
+	int			fastreg_cq_vector;
+	struct ib_cq		*fastreg_cq;
+	struct ib_wc            fastreg_wc[RDS_WC_MAX];
+	struct ib_qp		*fastreg_qp;
+	struct tasklet_struct	fastreg_tasklet;
+	atomic_t		fastreg_wrs;
+	struct rw_semaphore	fastreg_lock;
+	struct work_struct	fastreg_reset_w;
+
 	struct ib_mr		*mr;
 	struct rds_ib_mr_pool	*mr_1m_pool;
 	struct rds_ib_mr_pool   *mr_8k_pool;
@@ -586,6 +598,9 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn,
 				struct rdma_cm_event *event);
 void rds_ib_init_frag(unsigned int version);
 void rds_ib_conn_destroy_init(struct rds_connection *conn);
+void rds_ib_destroy_fastreg(struct rds_ib_device *rds_ibdev);
+int rds_ib_setup_fastreg(struct rds_ib_device *rds_ibdev);
+void rds_ib_reset_fastreg(struct work_struct *work);
 
 /* ib_rdma.c */
 int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev, __be32 ipaddr);
@@ -603,6 +618,7 @@ void rds_ib_free_mr(void *trans_private, int invalidate);
 void rds_ib_flush_mrs(void);
 int rds_ib_fmr_init(void);
 void rds_ib_fmr_exit(void);
+void rds_ib_fcq_handler(struct rds_ib_device *rds_ibdev, struct ib_wc *wc);
 void rds_ib_mr_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc);
 
 /* ib_recv.c */