Skip to content

Commit 46a94c1

Browse files
gerd-rauschjfvogel
gerd-rausch
authored and
jfvogel
committed
net/rds: Bring tasklets back for better latency
Re-introduce tasklets into the fastpath, as they yield better latency than workqueues. Tasklets are optional though and controlled by enabling option "tasklet" in module parameter "rds_ib_preferred_cpu", which is on by default. Orabug: 33440594 Signed-off-by: Gerd Rausch <[email protected]> Reviewed-by: William Kucharski <[email protected]> Orabug: 33590097 UEK6 => UEK7 (cherry picked from commit 43aa0f1) cherry-pick-repo=UEK/production/linux-uek.git Signed-off-by: Gerd Rausch <[email protected]> Reviewed-by: William Kucharski <[email protected]>
1 parent 71d2f06 commit 46a94c1

File tree

3 files changed

+81
-20
lines changed

3 files changed

+81
-20
lines changed

‎net/rds/ib.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,9 @@
7070
enum rds_ib_preferred_cpu_options {
7171
RDS_IB_PREFER_CPU_CQ = 1 << 0,
7272
RDS_IB_PREFER_CPU_NUMA = 1 << 1,
73-
RDS_IB_PREFER_CPU_DEFAULT = RDS_IB_PREFER_CPU_CQ
73+
RDS_IB_PREFER_CPU_TASKLET = 1 << 2,
74+
RDS_IB_PREFER_CPU_DEFAULT = RDS_IB_PREFER_CPU_CQ |
75+
RDS_IB_PREFER_CPU_TASKLET
7476
};
7577

7678
extern enum rds_ib_preferred_cpu_options rds_ib_preferred_cpu;
@@ -267,6 +269,8 @@ struct rds_ib_connection {
267269
atomic_t i_fastreg_wrs;
268270

269271
/* interrupt handling */
272+
struct tasklet_struct i_stasklet;
273+
struct tasklet_struct i_rtasklet;
270274
struct work_struct i_recv_w;
271275
struct work_struct i_send_w;
272276

@@ -448,6 +452,7 @@ struct rds_ib_device {
448452
struct ib_qp *fastreg_qp;
449453
atomic_t fastreg_wrs;
450454
struct rw_semaphore fastreg_lock;
455+
struct tasklet_struct fastreg_tasklet;
451456
struct work_struct fastreg_w;
452457
struct work_struct fastreg_reset_w;
453458

‎net/rds/ib_cm.c

Lines changed: 72 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -540,8 +540,11 @@ static void rds_ib_cq_comp_handler_fastreg(struct ib_cq *cq, void *context)
540540
{
541541
struct rds_ib_device *rds_ibdev = context;
542542

543-
queue_work_on(smp_processor_id(),
544-
rds_evt_wq, &rds_ibdev->fastreg_w);
543+
if (rds_ib_preferred_cpu & RDS_IB_PREFER_CPU_TASKLET)
544+
tasklet_schedule(&rds_ibdev->fastreg_tasklet);
545+
else
546+
queue_work_on(smp_processor_id(),
547+
rds_evt_wq, &rds_ibdev->fastreg_w);
545548
}
546549

547550

@@ -613,8 +616,11 @@ static void rds_ib_cq_comp_handler_send(struct ib_cq *cq, void *context)
613616
&ic->i_cq_follow_affinity_w,
614617
msecs_to_jiffies(RDS_IB_CQ_FOLLOW_AFFINITY_THROTTLE));
615618

616-
queue_work_on(ic->i_preferred_cpu,
617-
rds_evt_wq, &ic->i_send_w);
619+
if (rds_ib_preferred_cpu & RDS_IB_PREFER_CPU_TASKLET)
620+
tasklet_schedule(&ic->i_stasklet);
621+
else
622+
queue_work_on(ic->i_preferred_cpu,
623+
rds_evt_wq, &ic->i_send_w);
618624
}
619625

620626
static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
@@ -631,8 +637,11 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
631637
&ic->i_cq_follow_affinity_w,
632638
msecs_to_jiffies(RDS_IB_CQ_FOLLOW_AFFINITY_THROTTLE));
633639

634-
queue_work_on(ic->i_preferred_cpu,
635-
rds_evt_wq, &ic->i_recv_w);
640+
if (rds_ib_preferred_cpu & RDS_IB_PREFER_CPU_TASKLET)
641+
tasklet_schedule(&ic->i_rtasklet);
642+
else
643+
queue_work_on(ic->i_preferred_cpu,
644+
rds_evt_wq, &ic->i_recv_w);
636645
}
637646

638647
static void poll_fcq(struct rds_ib_device *rds_ibdev, struct ib_cq *cq,
@@ -699,6 +708,15 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
699708
}
700709
}
701710

711+
static void rds_ib_tasklet_fn_fastreg(unsigned long data)
712+
{
713+
struct rds_ib_device *rds_ibdev = (struct rds_ib_device *)data;
714+
715+
poll_fcq(rds_ibdev, rds_ibdev->fastreg_cq, rds_ibdev->fastreg_wc);
716+
ib_req_notify_cq(rds_ibdev->fastreg_cq, IB_CQ_NEXT_COMP);
717+
poll_fcq(rds_ibdev, rds_ibdev->fastreg_cq, rds_ibdev->fastreg_wc);
718+
}
719+
702720
static void rds_ib_cq_comp_handler_fastreg_w(struct work_struct *work)
703721
{
704722
struct rds_ib_device *rds_ibdev = container_of(work,
@@ -710,11 +728,8 @@ static void rds_ib_cq_comp_handler_fastreg_w(struct work_struct *work)
710728
poll_fcq(rds_ibdev, rds_ibdev->fastreg_cq, rds_ibdev->fastreg_wc);
711729
}
712730

713-
void rds_ib_send_w(struct work_struct *work)
731+
static void rds_ib_send_cb(struct rds_ib_connection *ic)
714732
{
715-
struct rds_ib_connection *ic = container_of(work,
716-
struct rds_ib_connection,
717-
i_send_w);
718733
struct rds_connection *conn = ic->conn;
719734

720735
rds_ib_stats_inc(s_ib_tasklet_call);
@@ -733,6 +748,22 @@ void rds_ib_send_w(struct work_struct *work)
733748
rds_send_xmit(&ic->conn->c_path[0]);
734749
}
735750

751+
void rds_ib_tasklet_fn_send(unsigned long data)
752+
{
753+
struct rds_ib_connection *ic = (struct rds_ib_connection *) data;
754+
755+
rds_ib_send_cb(ic);
756+
}
757+
758+
void rds_ib_send_w(struct work_struct *work)
759+
{
760+
struct rds_ib_connection *ic = container_of(work,
761+
struct rds_ib_connection,
762+
i_send_w);
763+
764+
rds_ib_send_cb(ic);
765+
}
766+
736767
/*
737768
* Note: rds_ib_rx(): don't call with irqs disabled.
738769
* It calls rds_send_drop_acked() which calls other
@@ -788,12 +819,8 @@ static void rds_ib_rx(struct rds_ib_connection *ic)
788819
}
789820
}
790821

791-
void rds_ib_recv_w(struct work_struct *work)
822+
static void rds_ib_recv_cb(struct rds_ib_connection *ic)
792823
{
793-
struct rds_ib_connection *ic = container_of(work,
794-
struct rds_ib_connection,
795-
i_recv_w);
796-
797824
spin_lock_bh(&ic->i_rx_lock);
798825
if (ic->i_rx_wait_for_handler)
799826
goto out;
@@ -802,6 +829,22 @@ void rds_ib_recv_w(struct work_struct *work)
802829
spin_unlock_bh(&ic->i_rx_lock);
803830
}
804831

832+
void rds_ib_tasklet_fn_recv(unsigned long data)
833+
{
834+
struct rds_ib_connection *ic = (struct rds_ib_connection *) data;
835+
836+
rds_ib_recv_cb(ic);
837+
}
838+
839+
void rds_ib_recv_w(struct work_struct *work)
840+
{
841+
struct rds_ib_connection *ic = container_of(work,
842+
struct rds_ib_connection,
843+
i_recv_w);
844+
845+
rds_ib_recv_cb(ic);
846+
}
847+
805848
static void rds_ib_rx_handler(struct work_struct *_work)
806849
{
807850
struct rds_ib_rx_work *work =
@@ -2208,8 +2251,11 @@ void rds_ib_conn_path_shutdown_prepare(struct rds_conn_path *cp)
22082251
the recv work to drain the RX CQ.
22092252
*/
22102253
wait_for_completion(&ic->i_last_wqe_complete);
2211-
queue_work_on(smp_processor_id(),
2212-
rds_evt_wq, &ic->i_recv_w);
2254+
if (rds_ib_preferred_cpu & RDS_IB_PREFER_CPU_TASKLET)
2255+
tasklet_schedule(&ic->i_rtasklet);
2256+
else
2257+
queue_work_on(smp_processor_id(),
2258+
rds_evt_wq, &ic->i_recv_w);
22132259
}
22142260
}
22152261
}
@@ -2249,6 +2295,8 @@ void rds_ib_conn_path_shutdown_final(struct rds_conn_path *cp)
22492295
if (ic->i_cm_id) {
22502296
cancel_delayed_work_sync(&ic->i_rx_w.work);
22512297

2298+
tasklet_kill(&ic->i_stasklet);
2299+
tasklet_kill(&ic->i_rtasklet);
22522300
flush_workqueue(rds_evt_wq);
22532301

22542302
ic->i_flags &= ~RDS_IB_CQ_ERR;
@@ -2362,6 +2410,8 @@ int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp)
23622410
INIT_LIST_HEAD(&ic->ib_node);
23632411
init_rwsem(&ic->i_cm_id_free_lock);
23642412

2413+
tasklet_init(&ic->i_stasklet, rds_ib_tasklet_fn_send, (unsigned long) ic);
2414+
tasklet_init(&ic->i_rtasklet, rds_ib_tasklet_fn_recv, (unsigned long) ic);
23652415
INIT_WORK(&ic->i_send_w, rds_ib_send_w);
23662416
INIT_WORK(&ic->i_recv_w, rds_ib_recv_w);
23672417
mutex_init(&ic->i_recv_mutex);
@@ -2451,7 +2501,9 @@ void rds_ib_destroy_fastreg(struct rds_ib_device *rds_ibdev)
24512501
*/
24522502
WARN_ON(atomic_read(&rds_ibdev->fastreg_wrs) != RDS_IB_DEFAULT_FREG_WR);
24532503

2504+
tasklet_kill(&rds_ibdev->fastreg_tasklet);
24542505
flush_workqueue(rds_evt_wq);
2506+
24552507
if (rds_ibdev->fastreg_qp) {
24562508
/* Destroy qp */
24572509
if (ib_destroy_qp(rds_ibdev->fastreg_qp))
@@ -2612,6 +2664,9 @@ int rds_ib_setup_fastreg(struct rds_ib_device *rds_ibdev)
26122664
"moved qp to RTS state for device",
26132665
0);
26142666

2667+
tasklet_init(&rds_ibdev->fastreg_tasklet, rds_ib_tasklet_fn_fastreg,
2668+
(unsigned long)rds_ibdev);
2669+
26152670
INIT_WORK(&rds_ibdev->fastreg_w, rds_ib_cq_comp_handler_fastreg_w);
26162671
atomic_set(&rds_ibdev->fastreg_wrs, RDS_IB_DEFAULT_FREG_WR);
26172672

‎net/rds/ib_rdma.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,9 @@ static struct {
4545
const char *name;
4646
enum rds_ib_preferred_cpu_options option;
4747
} rds_ib_preferred_cpu_options[] = {
48-
{ "cq", RDS_IB_PREFER_CPU_CQ },
49-
{ "numa", RDS_IB_PREFER_CPU_NUMA },
48+
{ "cq", RDS_IB_PREFER_CPU_CQ },
49+
{ "numa", RDS_IB_PREFER_CPU_NUMA },
50+
{ "tasklet", RDS_IB_PREFER_CPU_TASKLET },
5051
};
5152

5253
enum rds_ib_preferred_cpu_options rds_ib_preferred_cpu = RDS_IB_PREFER_CPU_DEFAULT;

0 commit comments

Comments
 (0)