net/rds: Use DMA memory pool allocation for rds_header

Currently, RDS calls ib_dma_alloc_coherent() to allocate a large piece
of contiguous DMA coherent memory to store struct rds_header for
sending/receiving packets.  The memory allocated is then partitioned
into struct rds_header.  This is not necessary and can be costly at
times when memory is fragmented.  Instead, RDS should use the DMA
memory pool interface to handle this.  This interface is to be used
for allocating small DMA coherent memory buffers like struct
rds_header.

Orabug: 28388601

Tested-by: Gerald Gibson <[email protected]>
Tested-by: Shih-Yu Huang <[email protected]>
Tested-by: Jindrizka Dominguez <[email protected]>
Suggested-by: Håkon Bugge <[email protected]>
Signed-off-by: Ka-Cheong Poon <[email protected]>
Reviewed-by: Santosh Shilimkar <[email protected]>
Reviewed-by: Matthew Wilcox (Oracle) <[email protected]>
Signed-off-by: Somasundaram Krishnasamy <[email protected]>

Author: kcp-git

net/rds/ib.c

@@ -131,6 +131,8 @@ static void rds_ib_dev_free_dev(struct rds_ib_device *rds_ibdev)
 		ib_dereg_mr(rds_ibdev->mr);
 	if (rds_ibdev->pd)
 		ib_dealloc_pd(rds_ibdev->pd);
+	if (rds_ibdev->rid_hdrs_pool)
+		dma_pool_destroy(rds_ibdev->rid_hdrs_pool);
 out:
 	mutex_unlock(&rds_ibdev->free_dev_lock);
 }
@@ -570,6 +572,12 @@ void rds_ib_add_one(struct ib_device *device)
 		rds_ibdev->pd = NULL;
 		goto put_dev;
 	}
+	rds_ibdev->rid_hdrs_pool = dma_pool_create(device->name,
+						   device->dma_device,
+						   sizeof(struct rds_header),
+						   L1_CACHE_BYTES, 0);
+	if (!rds_ibdev->rid_hdrs_pool)
+		goto put_dev;
 
 	rds_ibdev->vector_load = kzalloc(sizeof(int) *
 					device->num_comp_vectors, GFP_KERNEL);

net/rds/ib.h

@@ -210,8 +210,8 @@ struct rds_ib_connection {
 	/* tx */
 	struct rds_ib_work_ring	i_send_ring;
 	struct rm_data_op	*i_data_op;
-	struct rds_header	*i_send_hdrs;
-	u64			i_send_hdrs_dma;
+	struct rds_header	**i_send_hdrs;
+	dma_addr_t		*i_send_hdrs_dma;
 	struct rds_ib_send_work *i_sends;
 	atomic_t		i_signaled_sends;
 
@@ -221,8 +221,8 @@ struct rds_ib_connection {
 	struct rds_ib_work_ring	i_recv_ring;
 	struct rds_ib_incoming	*i_ibinc;
 	u32			i_recv_data_rem;
-	struct rds_header	*i_recv_hdrs;
-	u64			i_recv_hdrs_dma;
+	struct rds_header	**i_recv_hdrs;
+	dma_addr_t		*i_recv_hdrs_dma;
 	struct rds_ib_recv_work *i_recvs;
 	u64			i_ack_recv;	/* last ACK received */
 	struct rds_ib_refill_cache i_cache_incs;
@@ -307,8 +307,8 @@ struct rds_ib_srq {
 	struct ib_event_handler    s_event_handler;
 	struct rds_ib_recv_work    *s_recvs;
 	u32                        s_n_wr;
-	struct rds_header          *s_recv_hdrs;
-	u64                        s_recv_hdrs_dma;
+	struct rds_header          **s_recv_hdrs;
+	dma_addr_t                 *s_recv_hdrs_dma;
 	atomic_t                   s_num_posted;
 	unsigned long              s_refill_gate;
 	struct delayed_work        s_refill_w;
@@ -337,6 +337,7 @@ struct rds_ib_device {
 	struct list_head	conn_list;
 	struct ib_device	*dev;
 	struct ib_pd		*pd;
+	struct dma_pool		*rid_hdrs_pool; /* RDS headers DMA pool */
 
 	bool			use_fastreg;
 	int			fastreg_cq_vector;
@@ -592,6 +593,11 @@ u32 __rds_find_ifindex_v4(struct net *net, __be32 addr);
 #if IS_ENABLED(CONFIG_IPV6)
 u32 __rds_find_ifindex_v6(struct net *net, const struct in6_addr *addr);
 #endif
+struct rds_header **rds_dma_hdrs_alloc(struct ib_device *ibdev,
+				       struct dma_pool *pool,
+				       dma_addr_t **dma_addrs, u32 num_hdrs);
+void rds_dma_hdrs_free(struct dma_pool *pool, struct rds_header **hdrs,
+		       dma_addr_t *dma_addrs, u32 num_hdrs);
 
 /* ib_rdma.c */
 int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev,

net/rds/ib_cm.c

@@ -742,6 +742,68 @@ static void rds_ib_check_cq(struct ib_device *dev, struct rds_ib_device *rds_ibd
 		       spurious_completions, str, ctx);
 }
 
+/* Allocate DMA coherent memory to be used to store struct rds_header for
+ * sending/receiving packets.  The pointers to the DMA memory and the
+ * associated DMA addresses are stored in two arrays.
+ *
+ * @ibdev: the IB device
+ * @pool: the DMA memory pool
+ * @dma_addrs: pointer to the array for storing DMA addresses
+ * @num_hdrs: number of headers to allocate
+ *
+ * It returns the pointer to the array storing the DMA memory pointers.  On
+ * error, NULL pointer is returned.
+ */
+struct rds_header **rds_dma_hdrs_alloc(struct ib_device *ibdev,
+				       struct dma_pool *pool,
+				       dma_addr_t **dma_addrs, u32 num_hdrs)
+{
+	struct rds_header **hdrs;
+	dma_addr_t *hdr_daddrs;
+	u32 i;
+
+	hdrs = kvmalloc_node(sizeof(*hdrs) * num_hdrs, GFP_KERNEL,
+			     ibdev_to_node(ibdev));
+	if (!hdrs)
+		return NULL;
+
+	hdr_daddrs = kvmalloc_node(sizeof(*hdr_daddrs) * num_hdrs, GFP_KERNEL,
+				   ibdev_to_node(ibdev));
+	if (!hdr_daddrs) {
+		kvfree(hdrs);
+		return NULL;
+	}
+
+	for (i = 0; i < num_hdrs; i++) {
+		hdrs[i] = dma_pool_zalloc(pool, GFP_KERNEL, &hdr_daddrs[i]);
+		if (!hdrs[i]) {
+			rds_dma_hdrs_free(pool, hdrs, hdr_daddrs, i);
+			return NULL;
+		}
+	}
+
+	*dma_addrs = hdr_daddrs;
+	return hdrs;
+}
+
+/* Free the DMA memory used to store struct rds_header.
+ *
+ * @pool: the DMA memory pool
+ * @hdrs: pointer to the array storing DMA memory pointers
+ * @dma_addrs: pointer to the array storing DMA addresses
+ * @num_hdars: number of headers to free.
+ */
+void rds_dma_hdrs_free(struct dma_pool *pool, struct rds_header **hdrs,
+		       dma_addr_t *dma_addrs, u32 num_hdrs)
+{
+	u32 i;
+
+	for (i = 0; i < num_hdrs; i++)
+		dma_pool_free(pool, hdrs[i], dma_addrs[i]);
+	kvfree(hdrs);
+	kvfree(dma_addrs);
+}
+
 /*
  * This needs to be very careful to not leave IS_ERR pointers around for
  * cleanup to trip over.
@@ -752,6 +814,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
 	struct ib_device *dev = ic->i_cm_id->device;
 	struct ib_qp_init_attr qp_attr;
 	struct rds_ib_device *rds_ibdev;
+	struct dma_pool *pool;
 	int ret;
 	int mr_reg;
 
@@ -850,34 +913,33 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
 		goto out;
 	}
 
-	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
-					   ic->i_send_ring.w_nr *
-						sizeof(struct rds_header),
-					   &ic->i_send_hdrs_dma, GFP_KERNEL);
+	pool = rds_ibdev->rid_hdrs_pool;
+	ic->i_send_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_send_hdrs_dma,
+					     ic->i_send_ring.w_nr);
 	if (!ic->i_send_hdrs) {
 		ret = -ENOMEM;
-		rds_rtd(RDS_RTD_ERR, "ib_dma_alloc_coherent send failed\n");
+		rds_rtd(RDS_RTD_ERR, "%s: DMA send hdrs alloc failed\n",
+			__func__);
 		goto out;
 	}
 
 	if (!rds_ib_srq_enabled) {
-		ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
-					ic->i_recv_ring.w_nr *
-					sizeof(struct rds_header),
-					&ic->i_recv_hdrs_dma, GFP_KERNEL);
+		ic->i_recv_hdrs = rds_dma_hdrs_alloc(dev, pool,
+						     &ic->i_recv_hdrs_dma,
+						     ic->i_recv_ring.w_nr);
 		if (!ic->i_recv_hdrs) {
 			ret = -ENOMEM;
 			rds_rtd(RDS_RTD_ERR,
-				"ib_dma_alloc_coherent recv failed\n");
+				"%s: DMA recv hdrs alloc failed\n", __func__);
 			goto out;
 		}
 	}
 
-	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
-				       &ic->i_ack_dma, GFP_KERNEL);
+	ic->i_ack = dma_pool_zalloc(pool, GFP_KERNEL, &ic->i_ack_dma);
 	if (!ic->i_ack) {
 		ret = -ENOMEM;
-		rds_rtd(RDS_RTD_ERR, "ib_dma_alloc_coherent ack failed\n");
+		rds_rtd(RDS_RTD_ERR, "%s: DMA ack header alloc failed\n",
+			__func__);
 		goto out;
 	}
 
@@ -1467,8 +1529,6 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 		    conn, ic->i_cm_id, ic->i_pd, ic->i_rcq, ic->i_cm_id ? ic->i_cm_id->qp : NULL);
 
 	if (ic->i_cm_id) {
-		struct ib_device *dev = ic->i_cm_id->device;
-
 		rds_rtd_ptr(RDS_RTD_CM_EXT, "disconnecting conn %p cm_id %p\n", conn, ic->i_cm_id);
 		err = rdma_disconnect(ic->i_cm_id);
 		if (err) {
@@ -1517,24 +1577,39 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 		if (ic->i_cm_id->qp)
 			rdma_destroy_qp(ic->i_cm_id);
 
-		/* then free the resources that ib callbacks use */
-		if (ic->i_send_hdrs)
-			ib_dma_free_coherent(dev,
-					   ic->i_send_ring.w_nr *
-						sizeof(struct rds_header),
-					   ic->i_send_hdrs,
-					   ic->i_send_hdrs_dma);
-
-		if (ic->i_recv_hdrs)
-			ib_dma_free_coherent(dev,
-					   ic->i_recv_ring.w_nr *
-						sizeof(struct rds_header),
-					   ic->i_recv_hdrs,
-					   ic->i_recv_hdrs_dma);
-
-		if (ic->i_ack)
-			ib_dma_free_coherent(dev, sizeof(struct rds_header),
-					     ic->i_ack, ic->i_ack_dma);
+		if (ic->rds_ibdev) {
+			struct dma_pool *pool;
+
+			pool = ic->rds_ibdev->rid_hdrs_pool;
+
+			/* then free the resources that ib callbacks use */
+			if (ic->i_send_hdrs) {
+				rds_dma_hdrs_free(pool, ic->i_send_hdrs,
+						  ic->i_send_hdrs_dma,
+						  ic->i_send_ring.w_nr);
+				ic->i_send_hdrs = NULL;
+				ic->i_send_hdrs_dma = NULL;
+			}
+
+			if (ic->i_recv_hdrs) {
+				rds_dma_hdrs_free(pool, ic->i_recv_hdrs,
+						  ic->i_recv_hdrs_dma,
+						  ic->i_recv_ring.w_nr);
+				ic->i_recv_hdrs = NULL;
+				ic->i_recv_hdrs_dma = NULL;
+			}
+
+			if (ic->i_ack) {
+				dma_pool_free(pool, ic->i_ack, ic->i_ack_dma);
+				ic->i_ack = NULL;
+			}
+		} else {
+			WARN_ON(ic->i_send_hdrs);
+			WARN_ON(ic->i_send_hdrs_dma);
+			WARN_ON(ic->i_recv_hdrs);
+			WARN_ON(ic->i_recv_hdrs_dma);
+			WARN_ON(ic->i_ack);
+		}
 
 		if (ic->i_sends)
 			rds_ib_send_clear_ring(ic);
@@ -1552,9 +1627,6 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 		ic->i_cm_id = NULL;
 		ic->i_pd = NULL;
 		ic->i_mr = NULL;
-		ic->i_send_hdrs = NULL;
-		ic->i_recv_hdrs = NULL;
-		ic->i_ack = NULL;
 	}
 	BUG_ON(ic->rds_ibdev);
 

net/rds/ib_recv.c

@@ -77,7 +77,7 @@ void rds_ib_recv_init_ring(struct rds_ib_connection *ic)
 		recv->r_wr.num_sge = num_send_sge;
 
 		sge = recv->r_sge;
-		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));
+		sge->addr = ic->i_recv_hdrs_dma[i];
 		sge->length = sizeof(struct rds_header);
 		sge->lkey = ic->i_mr->lkey;
 
@@ -445,7 +445,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
 			    ic->i_frag_pages, DMA_FROM_DEVICE);
 
 	sge = recv->r_sge;
-	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);
+	sge->addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs];
 	sge->length = sizeof(struct rds_header);
 
 	for_each_sg(recv->r_frag->f_sg, sg, ic->i_frag_pages, i) {
@@ -517,10 +517,7 @@ static int rds_ib_srq_refill_one(struct rds_ib_srq *srq,
 			    ic->i_frag_pages, DMA_FROM_DEVICE);
 
 	sge = recv->r_sge;
-
-	sge->addr = srq->s_recv_hdrs_dma +
-		(recv - srq->s_recvs) *
-		sizeof(struct rds_header);
+	sge->addr = srq->s_recv_hdrs_dma[recv - srq->s_recvs];
 
 	sge->length = sizeof(struct rds_header);
 
@@ -586,9 +583,8 @@ static int rds_ib_srq_prefill_one(struct rds_ib_device *rds_ibdev,
 			    num_sge, DMA_FROM_DEVICE);
 
 	sge = &recv->r_sge[0];
-	sge->addr = rds_ibdev->srq->s_recv_hdrs_dma +
-			(recv - rds_ibdev->srq->s_recvs) *
-			sizeof(struct rds_header);
+	sge->addr = rds_ibdev->srq->s_recv_hdrs_dma[recv -
+						    rds_ibdev->srq->s_recvs];
 	sge->length = sizeof(struct rds_header);
 	sge->lkey = rds_ibdev->mr->lkey;
 
@@ -1202,7 +1198,7 @@ static void rds_ib_process_recv(struct rds_connection *conn,
 	}
 	data_len -= sizeof(struct rds_header);
 
-	ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs];
+	ihdr = ic->i_recv_hdrs[recv - ic->i_recvs];
 
 	/* Validate the checksum. */
 	if (!rds_message_verify_checksum(ihdr)) {
@@ -1326,7 +1322,7 @@ void rds_ib_srq_process_recv(struct rds_connection *conn,
 	}
 	data_len -= sizeof(struct rds_header);
 
-	ihdr = &ic->rds_ibdev->srq->s_recv_hdrs[recv->r_wr.wr_id];
+	ihdr = ic->rds_ibdev->srq->s_recv_hdrs[recv->r_wr.wr_id];
 
 	/* Validate the checksum. */
 	if (!rds_message_verify_checksum(ihdr)) {
@@ -1679,6 +1675,7 @@ int rds_ib_srq_init(struct rds_ib_device *rds_ibdev)
 			.max_sge = rds_ibdev->max_sge
 		}
 	};
+	struct rds_header **hdrs;
 
 	/* This is called in two paths
 	 * 1) during insmod of rds_rdma module
@@ -1707,14 +1704,15 @@ int rds_ib_srq_init(struct rds_ib_device *rds_ibdev)
 		return 1;
 	}
 
-	rds_ibdev->srq->s_recv_hdrs = ib_dma_alloc_coherent(rds_ibdev->dev,
-				rds_ibdev->srq->s_n_wr *
-				sizeof(struct rds_header),
-				&rds_ibdev->srq->s_recv_hdrs_dma, GFP_KERNEL);
-	if (!rds_ibdev->srq->s_recv_hdrs) {
-		printk(KERN_WARNING "ib_dma_alloc_coherent failed\n");
+	hdrs = rds_dma_hdrs_alloc(rds_ibdev->dev,
+				  rds_ibdev->rid_hdrs_pool,
+				  &rds_ibdev->srq->s_recv_hdrs_dma,
+				  rds_ibdev->srq->s_n_wr);
+	if (!hdrs) {
+		pr_warn("%s: DMA recv hdrs alloc failed\n", __func__);
 		return 1;
 	}
+	rds_ibdev->srq->s_recv_hdrs = hdrs;
 
 	rds_ibdev->srq->s_recvs = vmalloc(rds_ibdev->srq->s_n_wr *
 				sizeof(struct rds_ib_recv_work));
@@ -1752,11 +1750,10 @@ void rds_ib_srq_exit(struct rds_ib_device *rds_ibdev)
 	rds_ibdev->srq->s_srq = NULL;
 
 	if (rds_ibdev->srq->s_recv_hdrs)
-		ib_dma_free_coherent(rds_ibdev->dev,
-				rds_ibdev->srq->s_n_wr *
-				sizeof(struct rds_header),
-				rds_ibdev->srq->s_recv_hdrs,
-				rds_ibdev->srq->s_recv_hdrs_dma);
+		rds_dma_hdrs_free(rds_ibdev->rid_hdrs_pool,
+				  rds_ibdev->srq->s_recv_hdrs,
+				  rds_ibdev->srq->s_recv_hdrs_dma,
+				  rds_ibdev->srq->s_n_wr);
 
 	rds_ib_srq_clear_ring(rds_ibdev);
 	vfree(rds_ibdev->srq->s_recvs);