Skip to content

Commit d654788

Browse files
chuckleveramschuma-ntap
chucklever
authored and
amschuma-ntap
committed
xprtrdma: Make rpcrdma_{un}map_one() into inline functions
These functions are called in a loop for each page transferred via RDMA READ or WRITE. Extract loop invariants and inline them to reduce CPU overhead. Signed-off-by: Chuck Lever <[email protected]> Tested-by: Devesh Sharma <[email protected]> Tested-by: Meghana Cheripady <[email protected]> Tested-by: Veeresh U. Kokatnur <[email protected]> Signed-off-by: Anna Schumaker <[email protected]>
1 parent e46ac34 commit d654788

File tree

5 files changed

+73
-46
lines changed

5 files changed

+73
-46
lines changed

‎net/sunrpc/xprtrdma/fmr_ops.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
8585
int nsegs, bool writing)
8686
{
8787
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
88+
struct ib_device *device = ia->ri_id->device;
89+
enum dma_data_direction direction = rpcrdma_data_dir(writing);
8890
struct rpcrdma_mr_seg *seg1 = seg;
8991
struct rpcrdma_mw *mw = seg1->rl_mw;
9092
u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];
@@ -97,7 +99,7 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
9799
if (nsegs > RPCRDMA_MAX_FMR_SGES)
98100
nsegs = RPCRDMA_MAX_FMR_SGES;
99101
for (i = 0; i < nsegs;) {
100-
rpcrdma_map_one(ia, seg, writing);
102+
rpcrdma_map_one(device, seg, direction);
101103
physaddrs[i] = seg->mr_dma;
102104
len += seg->mr_len;
103105
++seg;
@@ -123,7 +125,7 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
123125
__func__, len, (unsigned long long)seg1->mr_dma,
124126
pageoff, i, rc);
125127
while (i--)
126-
rpcrdma_unmap_one(ia, --seg);
128+
rpcrdma_unmap_one(device, --seg);
127129
return rc;
128130
}
129131

@@ -135,14 +137,16 @@ fmr_op_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg)
135137
{
136138
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
137139
struct rpcrdma_mr_seg *seg1 = seg;
140+
struct ib_device *device;
138141
int rc, nsegs = seg->mr_nsegs;
139142
LIST_HEAD(l);
140143

141144
list_add(&seg1->rl_mw->r.fmr->list, &l);
142145
rc = ib_unmap_fmr(&l);
143146
read_lock(&ia->ri_qplock);
147+
device = ia->ri_id->device;
144148
while (seg1->mr_nsegs--)
145-
rpcrdma_unmap_one(ia, seg++);
149+
rpcrdma_unmap_one(device, seg++);
146150
read_unlock(&ia->ri_qplock);
147151
if (rc)
148152
goto out_err;

‎net/sunrpc/xprtrdma/frwr_ops.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
178178
int nsegs, bool writing)
179179
{
180180
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
181+
struct ib_device *device = ia->ri_id->device;
182+
enum dma_data_direction direction = rpcrdma_data_dir(writing);
181183
struct rpcrdma_mr_seg *seg1 = seg;
182184
struct rpcrdma_mw *mw = seg1->rl_mw;
183185
struct rpcrdma_frmr *frmr = &mw->r.frmr;
@@ -197,7 +199,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
197199
if (nsegs > ia->ri_max_frmr_depth)
198200
nsegs = ia->ri_max_frmr_depth;
199201
for (page_no = i = 0; i < nsegs;) {
200-
rpcrdma_map_one(ia, seg, writing);
202+
rpcrdma_map_one(device, seg, direction);
201203
pa = seg->mr_dma;
202204
for (seg_len = seg->mr_len; seg_len > 0; seg_len -= PAGE_SIZE) {
203205
frmr->fr_pgl->page_list[page_no++] = pa;
@@ -247,7 +249,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
247249
ib_update_fast_reg_key(mr, --key);
248250
frmr->fr_state = FRMR_IS_INVALID;
249251
while (i--)
250-
rpcrdma_unmap_one(ia, --seg);
252+
rpcrdma_unmap_one(device, --seg);
251253
return rc;
252254
}
253255

@@ -261,6 +263,7 @@ frwr_op_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg)
261263
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
262264
struct ib_send_wr invalidate_wr, *bad_wr;
263265
int rc, nsegs = seg->mr_nsegs;
266+
struct ib_device *device;
264267

265268
seg1->rl_mw->r.frmr.fr_state = FRMR_IS_INVALID;
266269

@@ -271,8 +274,9 @@ frwr_op_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg)
271274
DECR_CQCOUNT(&r_xprt->rx_ep);
272275

273276
read_lock(&ia->ri_qplock);
277+
device = ia->ri_id->device;
274278
while (seg1->mr_nsegs--)
275-
rpcrdma_unmap_one(ia, seg++);
279+
rpcrdma_unmap_one(device, seg++);
276280
rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr);
277281
read_unlock(&ia->ri_qplock);
278282
if (rc)

‎net/sunrpc/xprtrdma/physical_ops.c

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ physical_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
5050
{
5151
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
5252

53-
rpcrdma_map_one(ia, seg, writing);
53+
rpcrdma_map_one(ia->ri_id->device, seg,
54+
rpcrdma_data_dir(writing));
5455
seg->mr_rkey = ia->ri_bind_mem->rkey;
5556
seg->mr_base = seg->mr_dma;
5657
seg->mr_nsegs = 1;
@@ -62,7 +63,12 @@ physical_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
6263
static int
6364
physical_op_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg)
6465
{
65-
rpcrdma_unmap_one(&r_xprt->rx_ia, seg);
66+
struct rpcrdma_ia *ia = &r_xprt->rx_ia;
67+
68+
read_lock(&ia->ri_qplock);
69+
rpcrdma_unmap_one(ia->ri_id->device, seg);
70+
read_unlock(&ia->ri_qplock);
71+
6672
return 1;
6773
}
6874

‎net/sunrpc/xprtrdma/verbs.c

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,6 +1436,14 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
14361436
* Wrappers for internal-use kmalloc memory registration, used by buffer code.
14371437
*/
14381438

1439+
void
1440+
rpcrdma_mapping_error(struct rpcrdma_mr_seg *seg)
1441+
{
1442+
dprintk("RPC: map_one: offset %p iova %llx len %zu\n",
1443+
seg->mr_offset,
1444+
(unsigned long long)seg->mr_dma, seg->mr_dmalen);
1445+
}
1446+
14391447
static int
14401448
rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,
14411449
struct ib_mr **mrp, struct ib_sge *iov)
@@ -1560,42 +1568,6 @@ rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
15601568
}
15611569
}
15621570

1563-
/*
1564-
* Wrappers for chunk registration, shared by read/write chunk code.
1565-
*/
1566-
1567-
void
1568-
rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, bool writing)
1569-
{
1570-
seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1571-
seg->mr_dmalen = seg->mr_len;
1572-
if (seg->mr_page)
1573-
seg->mr_dma = ib_dma_map_page(ia->ri_id->device,
1574-
seg->mr_page, offset_in_page(seg->mr_offset),
1575-
seg->mr_dmalen, seg->mr_dir);
1576-
else
1577-
seg->mr_dma = ib_dma_map_single(ia->ri_id->device,
1578-
seg->mr_offset,
1579-
seg->mr_dmalen, seg->mr_dir);
1580-
if (ib_dma_mapping_error(ia->ri_id->device, seg->mr_dma)) {
1581-
dprintk("RPC: %s: mr_dma %llx mr_offset %p mr_dma_len %zu\n",
1582-
__func__,
1583-
(unsigned long long)seg->mr_dma,
1584-
seg->mr_offset, seg->mr_dmalen);
1585-
}
1586-
}
1587-
1588-
void
1589-
rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg)
1590-
{
1591-
if (seg->mr_page)
1592-
ib_dma_unmap_page(ia->ri_id->device,
1593-
seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1594-
else
1595-
ib_dma_unmap_single(ia->ri_id->device,
1596-
seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1597-
}
1598-
15991571
/*
16001572
* Prepost any receive buffer, then post send.
16011573
*

‎net/sunrpc/xprtrdma/xprt_rdma.h

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -424,8 +424,49 @@ void rpcrdma_free_regbuf(struct rpcrdma_ia *,
424424
struct rpcrdma_regbuf *);
425425

426426
unsigned int rpcrdma_max_segments(struct rpcrdma_xprt *);
427-
void rpcrdma_map_one(struct rpcrdma_ia *, struct rpcrdma_mr_seg *, bool);
428-
void rpcrdma_unmap_one(struct rpcrdma_ia *, struct rpcrdma_mr_seg *);
427+
428+
/*
429+
* Wrappers for chunk registration, shared by read/write chunk code.
430+
*/
431+
432+
void rpcrdma_mapping_error(struct rpcrdma_mr_seg *);
433+
434+
static inline enum dma_data_direction
435+
rpcrdma_data_dir(bool writing)
436+
{
437+
return writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
438+
}
439+
440+
static inline void
441+
rpcrdma_map_one(struct ib_device *device, struct rpcrdma_mr_seg *seg,
442+
enum dma_data_direction direction)
443+
{
444+
seg->mr_dir = direction;
445+
seg->mr_dmalen = seg->mr_len;
446+
447+
if (seg->mr_page)
448+
seg->mr_dma = ib_dma_map_page(device,
449+
seg->mr_page, offset_in_page(seg->mr_offset),
450+
seg->mr_dmalen, seg->mr_dir);
451+
else
452+
seg->mr_dma = ib_dma_map_single(device,
453+
seg->mr_offset,
454+
seg->mr_dmalen, seg->mr_dir);
455+
456+
if (ib_dma_mapping_error(device, seg->mr_dma))
457+
rpcrdma_mapping_error(seg);
458+
}
459+
460+
static inline void
461+
rpcrdma_unmap_one(struct ib_device *device, struct rpcrdma_mr_seg *seg)
462+
{
463+
if (seg->mr_page)
464+
ib_dma_unmap_page(device,
465+
seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
466+
else
467+
ib_dma_unmap_single(device,
468+
seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
469+
}
429470

430471
/*
431472
* RPC/RDMA connection management calls - xprtrdma/rpc_rdma.c

0 commit comments

Comments
 (0)