Skip to content

Commit 32467c4

Browse files
Sagi Grimbergdledford
Sagi Grimberg
authored and
dledford
committed
IB/iser: Unify fast memory registration flows
iser_reg_rdma_mem_[fastreg|fmr] share a lot of code, and logically do the same thing other than the buffer registration method itself (iser_fast_reg_mr vs. iser_fast_reg_fmr). The DIF logic is not implemented in the FMR flow as there is no existing device that supports FMRs and Signature feature. This patch unifies the flow in a single routine iser_reg_rdma_mem and just split to fmr/frwr for the buffer registration itself. Also, for symmetry reasons, unify iser_unreg_rdma_mem (which will call the relevant device specific unreg routine). Signed-off-by: Sagi Grimberg <[email protected]> Signed-off-by: Adir Lev <[email protected]> Signed-off-by: Doug Ledford <[email protected]>
1 parent 8172290 commit 32467c4

File tree

3 files changed

+113
-131
lines changed

3 files changed

+113
-131
lines changed

‎drivers/infiniband/ulp/iser/iscsi_iser.h

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ struct iser_data_buf {
239239
struct iser_device;
240240
struct iscsi_iser_task;
241241
struct iscsi_endpoint;
242+
struct iser_reg_resources;
242243

243244
/**
244245
* struct iser_mem_reg - iSER memory registration info
@@ -331,19 +332,21 @@ struct iser_comp {
331332
*
332333
* @alloc_reg_res: Allocate registration resources
333334
* @free_reg_res: Free registration resources
334-
* @reg_rdma_mem: Register memory buffers
335-
* @unreg_rdma_mem: Un-register memory buffers
335+
* @fast_reg_mem: Register memory buffers
336+
* @unreg_mem: Un-register memory buffers
336337
* @reg_desc_get: Get a registration descriptor for pool
337338
* @reg_desc_put: Get a registration descriptor to pool
338339
*/
339340
struct iser_reg_ops {
340341
int (*alloc_reg_res)(struct ib_conn *ib_conn,
341342
unsigned cmds_max);
342343
void (*free_reg_res)(struct ib_conn *ib_conn);
343-
int (*reg_rdma_mem)(struct iscsi_iser_task *iser_task,
344-
enum iser_data_dir cmd_dir);
345-
void (*unreg_rdma_mem)(struct iscsi_iser_task *iser_task,
346-
enum iser_data_dir cmd_dir);
344+
int (*reg_mem)(struct iscsi_iser_task *iser_task,
345+
struct iser_data_buf *mem,
346+
struct iser_reg_resources *rsc,
347+
struct iser_mem_reg *reg);
348+
void (*unreg_mem)(struct iscsi_iser_task *iser_task,
349+
enum iser_data_dir cmd_dir);
347350
struct iser_fr_desc * (*reg_desc_get)(struct ib_conn *ib_conn);
348351
void (*reg_desc_put)(struct ib_conn *ib_conn,
349352
struct iser_fr_desc *desc);
@@ -622,10 +625,10 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
622625
struct iser_data_buf *mem,
623626
enum iser_data_dir cmd_dir);
624627

625-
int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
626-
enum iser_data_dir cmd_dir);
627-
int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task,
628-
enum iser_data_dir cmd_dir);
628+
int iser_reg_rdma_mem(struct iscsi_iser_task *task,
629+
enum iser_data_dir dir);
630+
void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
631+
enum iser_data_dir dir);
629632

630633
int iser_connect(struct iser_conn *iser_conn,
631634
struct sockaddr *src_addr,

‎drivers/infiniband/ulp/iser/iser_initiator.c

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
4949

5050
{
5151
struct iscsi_iser_task *iser_task = task->dd_data;
52-
struct iser_device *device = iser_task->iser_conn->ib_conn.device;
5352
struct iser_mem_reg *mem_reg;
5453
int err;
5554
struct iser_hdr *hdr = &iser_task->desc.iser_header;
@@ -73,7 +72,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
7372
return err;
7473
}
7574

76-
err = device->reg_ops->reg_rdma_mem(iser_task, ISER_DIR_IN);
75+
err = iser_reg_rdma_mem(iser_task, ISER_DIR_IN);
7776
if (err) {
7877
iser_err("Failed to set up Data-IN RDMA\n");
7978
return err;
@@ -103,7 +102,6 @@ iser_prepare_write_cmd(struct iscsi_task *task,
103102
unsigned int edtl)
104103
{
105104
struct iscsi_iser_task *iser_task = task->dd_data;
106-
struct iser_device *device = iser_task->iser_conn->ib_conn.device;
107105
struct iser_mem_reg *mem_reg;
108106
int err;
109107
struct iser_hdr *hdr = &iser_task->desc.iser_header;
@@ -128,7 +126,7 @@ iser_prepare_write_cmd(struct iscsi_task *task,
128126
return err;
129127
}
130128

131-
err = device->reg_ops->reg_rdma_mem(iser_task, ISER_DIR_OUT);
129+
err = iser_reg_rdma_mem(iser_task, ISER_DIR_OUT);
132130
if (err != 0) {
133131
iser_err("Failed to register write cmd RDMA mem\n");
134132
return err;
@@ -662,7 +660,6 @@ void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
662660

663661
void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
664662
{
665-
struct iser_device *device = iser_task->iser_conn->ib_conn.device;
666663
int is_rdma_data_aligned = 1;
667664
int is_rdma_prot_aligned = 1;
668665
int prot_count = scsi_prot_sg_count(iser_task->sc);
@@ -699,7 +696,7 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
699696
}
700697

701698
if (iser_task->dir[ISER_DIR_IN]) {
702-
device->reg_ops->unreg_rdma_mem(iser_task, ISER_DIR_IN);
699+
iser_unreg_rdma_mem(iser_task, ISER_DIR_IN);
703700
if (is_rdma_data_aligned)
704701
iser_dma_unmap_task_data(iser_task,
705702
&iser_task->data[ISER_DIR_IN],
@@ -711,7 +708,7 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
711708
}
712709

713710
if (iser_task->dir[ISER_DIR_OUT]) {
714-
device->reg_ops->unreg_rdma_mem(iser_task, ISER_DIR_OUT);
711+
iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT);
715712
if (is_rdma_data_aligned)
716713
iser_dma_unmap_task_data(iser_task,
717714
&iser_task->data[ISER_DIR_OUT],

‎drivers/infiniband/ulp/iser/iser_memory.c

Lines changed: 96 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -38,21 +38,31 @@
3838
#include <linux/scatterlist.h>
3939

4040
#include "iscsi_iser.h"
41+
static
42+
int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
43+
struct iser_data_buf *mem,
44+
struct iser_reg_resources *rsc,
45+
struct iser_mem_reg *mem_reg);
46+
static
47+
int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
48+
struct iser_data_buf *mem,
49+
struct iser_reg_resources *rsc,
50+
struct iser_mem_reg *mem_reg);
4151

4252
static struct iser_reg_ops fastreg_ops = {
4353
.alloc_reg_res = iser_alloc_fastreg_pool,
4454
.free_reg_res = iser_free_fastreg_pool,
45-
.reg_rdma_mem = iser_reg_rdma_mem_fastreg,
46-
.unreg_rdma_mem = iser_unreg_mem_fastreg,
55+
.reg_mem = iser_fast_reg_mr,
56+
.unreg_mem = iser_unreg_mem_fastreg,
4757
.reg_desc_get = iser_reg_desc_get_fr,
4858
.reg_desc_put = iser_reg_desc_put_fr,
4959
};
5060

5161
static struct iser_reg_ops fmr_ops = {
5262
.alloc_reg_res = iser_alloc_fmr_pool,
5363
.free_reg_res = iser_free_fmr_pool,
54-
.reg_rdma_mem = iser_reg_rdma_mem_fmr,
55-
.unreg_rdma_mem = iser_unreg_mem_fmr,
64+
.reg_mem = iser_fast_reg_fmr,
65+
.unreg_mem = iser_unreg_mem_fmr,
5666
.reg_desc_get = iser_reg_desc_get_fmr,
5767
.reg_desc_put = iser_reg_desc_put_fmr,
5868
};
@@ -574,62 +584,6 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
574584
reg->mem_h = NULL;
575585
}
576586

577-
/**
578-
* iser_reg_rdma_mem_fmr - Registers memory intended for RDMA,
579-
* using FMR (if possible) obtaining rkey and va
580-
*
581-
* returns 0 on success, errno code on failure
582-
*/
583-
int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
584-
enum iser_data_dir cmd_dir)
585-
{
586-
struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
587-
struct iser_device *device = ib_conn->device;
588-
struct ib_device *ibdev = device->ib_device;
589-
struct iser_data_buf *mem = &iser_task->data[cmd_dir];
590-
struct iser_mem_reg *mem_reg;
591-
int aligned_len;
592-
int err;
593-
int i;
594-
595-
mem_reg = &iser_task->rdma_reg[cmd_dir];
596-
597-
aligned_len = iser_data_buf_aligned_len(mem, ibdev);
598-
if (aligned_len != mem->dma_nents) {
599-
err = fall_to_bounce_buf(iser_task, mem, cmd_dir);
600-
if (err) {
601-
iser_err("failed to allocate bounce buffer\n");
602-
return err;
603-
}
604-
}
605-
606-
/* if there a single dma entry, FMR is not needed */
607-
if (mem->dma_nents == 1) {
608-
return iser_reg_dma(device, mem, mem_reg);
609-
} else { /* use FMR for multiple dma entries */
610-
struct iser_fr_desc *desc;
611-
612-
desc = device->reg_ops->reg_desc_get(ib_conn);
613-
err = iser_fast_reg_fmr(iser_task, mem, &desc->rsc, mem_reg);
614-
if (err && err != -EAGAIN) {
615-
iser_data_buf_dump(mem, ibdev);
616-
iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
617-
mem->dma_nents,
618-
ntoh24(iser_task->desc.iscsi_header.dlength));
619-
iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
620-
desc->rsc.page_vec->data_size,
621-
desc->rsc.page_vec->length,
622-
desc->rsc.page_vec->offset);
623-
for (i = 0; i < desc->rsc.page_vec->length; i++)
624-
iser_err("page_vec[%d] = 0x%llx\n", i,
625-
(unsigned long long)desc->rsc.page_vec->pages[i]);
626-
}
627-
if (err)
628-
return err;
629-
}
630-
return 0;
631-
}
632-
633587
static void
634588
iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs,
635589
struct ib_sig_domain *domain)
@@ -775,19 +729,12 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
775729
{
776730
struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
777731
struct iser_device *device = ib_conn->device;
778-
struct ib_mr *mr;
779-
struct ib_fast_reg_page_list *frpl;
732+
struct ib_mr *mr = rsc->mr;
733+
struct ib_fast_reg_page_list *frpl = rsc->frpl;
780734
struct ib_send_wr fastreg_wr, inv_wr;
781735
struct ib_send_wr *bad_wr, *wr = NULL;
782736
int ret, offset, size, plen;
783737

784-
/* if there a single dma entry, dma mr suffices */
785-
if (mem->dma_nents == 1)
786-
return iser_reg_dma(device, mem, reg);
787-
788-
mr = rsc->mr;
789-
frpl = rsc->frpl;
790-
791738
plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list,
792739
&offset, &size);
793740
if (plen * SIZE_4K < size) {
@@ -834,78 +781,113 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
834781
return ret;
835782
}
836783

837-
/**
838-
* iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA,
839-
* using Fast Registration WR (if possible) obtaining rkey and va
840-
*
841-
* returns 0 on success, errno code on failure
842-
*/
843-
int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task,
844-
enum iser_data_dir cmd_dir)
784+
static int
785+
iser_handle_unaligned_buf(struct iscsi_iser_task *task,
786+
struct iser_data_buf *mem,
787+
enum iser_data_dir dir)
845788
{
846-
struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
847-
struct iser_device *device = ib_conn->device;
848-
struct ib_device *ibdev = device->ib_device;
849-
struct iser_data_buf *mem = &iser_task->data[cmd_dir];
850-
struct iser_mem_reg *mem_reg = &iser_task->rdma_reg[cmd_dir];
851-
struct iser_fr_desc *desc = NULL;
789+
struct iser_conn *iser_conn = task->iser_conn;
790+
struct iser_device *device = iser_conn->ib_conn.device;
852791
int err, aligned_len;
853792

854-
aligned_len = iser_data_buf_aligned_len(mem, ibdev);
793+
aligned_len = iser_data_buf_aligned_len(mem, device->ib_device);
855794
if (aligned_len != mem->dma_nents) {
856-
err = fall_to_bounce_buf(iser_task, mem, cmd_dir);
857-
if (err) {
858-
iser_err("failed to allocate bounce buffer\n");
795+
err = fall_to_bounce_buf(task, mem, dir);
796+
if (err)
859797
return err;
860-
}
861798
}
862799

800+
return 0;
801+
}
802+
803+
static int
804+
iser_reg_prot_sg(struct iscsi_iser_task *task,
805+
struct iser_data_buf *mem,
806+
struct iser_fr_desc *desc,
807+
struct iser_mem_reg *reg)
808+
{
809+
struct iser_device *device = task->iser_conn->ib_conn.device;
810+
811+
if (mem->dma_nents == 1)
812+
return iser_reg_dma(device, mem, reg);
813+
814+
return device->reg_ops->reg_mem(task, mem, &desc->pi_ctx->rsc, reg);
815+
}
816+
817+
static int
818+
iser_reg_data_sg(struct iscsi_iser_task *task,
819+
struct iser_data_buf *mem,
820+
struct iser_fr_desc *desc,
821+
struct iser_mem_reg *reg)
822+
{
823+
struct iser_device *device = task->iser_conn->ib_conn.device;
824+
825+
if (mem->dma_nents == 1)
826+
return iser_reg_dma(device, mem, reg);
827+
828+
return device->reg_ops->reg_mem(task, mem, &desc->rsc, reg);
829+
}
830+
831+
int iser_reg_rdma_mem(struct iscsi_iser_task *task,
832+
enum iser_data_dir dir)
833+
{
834+
struct ib_conn *ib_conn = &task->iser_conn->ib_conn;
835+
struct iser_device *device = ib_conn->device;
836+
struct iser_data_buf *mem = &task->data[dir];
837+
struct iser_mem_reg *reg = &task->rdma_reg[dir];
838+
struct iser_fr_desc *desc = NULL;
839+
int err;
840+
841+
err = iser_handle_unaligned_buf(task, mem, dir);
842+
if (unlikely(err))
843+
return err;
844+
863845
if (mem->dma_nents != 1 ||
864-
scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
846+
scsi_get_prot_op(task->sc) != SCSI_PROT_NORMAL) {
865847
desc = device->reg_ops->reg_desc_get(ib_conn);
866-
mem_reg->mem_h = desc;
848+
reg->mem_h = desc;
867849
}
868850

869-
err = iser_fast_reg_mr(iser_task, mem,
870-
desc ? &desc->rsc : NULL, mem_reg);
871-
if (err)
851+
err = iser_reg_data_sg(task, mem, desc, reg);
852+
if (unlikely(err))
872853
goto err_reg;
873854

874-
if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
855+
if (scsi_get_prot_op(task->sc) != SCSI_PROT_NORMAL) {
875856
struct iser_mem_reg prot_reg;
876857

877858
memset(&prot_reg, 0, sizeof(prot_reg));
878-
if (scsi_prot_sg_count(iser_task->sc)) {
879-
mem = &iser_task->prot[cmd_dir];
880-
aligned_len = iser_data_buf_aligned_len(mem, ibdev);
881-
if (aligned_len != mem->dma_nents) {
882-
err = fall_to_bounce_buf(iser_task, mem,
883-
cmd_dir);
884-
if (err) {
885-
iser_err("failed to allocate bounce buffer\n");
886-
return err;
887-
}
888-
}
859+
if (scsi_prot_sg_count(task->sc)) {
860+
mem = &task->prot[dir];
861+
err = iser_handle_unaligned_buf(task, mem, dir);
862+
if (unlikely(err))
863+
goto err_reg;
889864

890-
err = iser_fast_reg_mr(iser_task, mem,
891-
&desc->pi_ctx->rsc, &prot_reg);
892-
if (err)
865+
err = iser_reg_prot_sg(task, mem, desc, &prot_reg);
866+
if (unlikely(err))
893867
goto err_reg;
894868
}
895869

896-
err = iser_reg_sig_mr(iser_task, desc->pi_ctx, mem_reg,
897-
&prot_reg, mem_reg);
898-
if (err) {
899-
iser_err("Failed to register signature mr\n");
900-
return err;
901-
}
870+
err = iser_reg_sig_mr(task, desc->pi_ctx, reg,
871+
&prot_reg, reg);
872+
if (unlikely(err))
873+
goto err_reg;
874+
902875
desc->pi_ctx->sig_protected = 1;
903876
}
904877

905878
return 0;
879+
906880
err_reg:
907881
if (desc)
908882
device->reg_ops->reg_desc_put(ib_conn, desc);
909883

910884
return err;
911885
}
886+
887+
void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
888+
enum iser_data_dir dir)
889+
{
890+
struct iser_device *device = task->iser_conn->ib_conn.device;
891+
892+
device->reg_ops->unreg_mem(task, dir);
893+
}

0 commit comments

Comments
 (0)