Skip to content

Commit 3f6be0e

Browse files
davem330davem330
committed
Merge branch 'RDS-zerocopy-code-enhancements'
Sowmini Varadhan says: ==================== RDS: zerocopy code enhancements A couple of enhancements to the rds zerocop code - patch 1 refactors rds_message_copy_from_user to pull the zcopy logic into its own function - patch 2 drops the usage sk_buff to track MSG_ZEROCOPY cookies and uses a simple linked list (enhancement suggested by willemb during code review) ==================== Signed-off-by: David S. Miller <[email protected]>
2 parents c33b3b9 + 9426bbc commit 3f6be0e

File tree

4 files changed

+138
-85
lines changed

4 files changed

+138
-85
lines changed

‎net/rds/af_rds.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ static int rds_release(struct socket *sock)
7777
rds_send_drop_to(rs, NULL);
7878
rds_rdma_drop_keys(rs);
7979
rds_notify_queue_get(rs, NULL);
80-
__skb_queue_purge(&rs->rs_zcookie_queue);
80+
rds_notify_msg_zcopy_purge(&rs->rs_zcookie_queue);
8181

8282
spin_lock_bh(&rds_sock_lock);
8383
list_del_init(&rs->rs_item);
@@ -180,7 +180,7 @@ static __poll_t rds_poll(struct file *file, struct socket *sock,
180180
}
181181
if (!list_empty(&rs->rs_recv_queue) ||
182182
!list_empty(&rs->rs_notify_queue) ||
183-
!skb_queue_empty(&rs->rs_zcookie_queue))
183+
!list_empty(&rs->rs_zcookie_queue.zcookie_head))
184184
mask |= (EPOLLIN | EPOLLRDNORM);
185185
if (rs->rs_snd_bytes < rds_sk_sndbuf(rs))
186186
mask |= (EPOLLOUT | EPOLLWRNORM);
@@ -515,7 +515,7 @@ static int __rds_create(struct socket *sock, struct sock *sk, int protocol)
515515
INIT_LIST_HEAD(&rs->rs_recv_queue);
516516
INIT_LIST_HEAD(&rs->rs_notify_queue);
517517
INIT_LIST_HEAD(&rs->rs_cong_list);
518-
skb_queue_head_init(&rs->rs_zcookie_queue);
518+
rds_message_zcopy_queue_init(&rs->rs_zcookie_queue);
519519
spin_lock_init(&rs->rs_rdma_lock);
520520
rs->rs_rdma_keys = RB_ROOT;
521521
rs->rs_rx_traces = 0;

‎net/rds/message.c

Lines changed: 102 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,16 @@ static unsigned int rds_exthdr_size[__RDS_EXTHDR_MAX] = {
4848
[RDS_EXTHDR_GEN_NUM] = sizeof(u32),
4949
};
5050

51-
5251
void rds_message_addref(struct rds_message *rm)
5352
{
5453
rdsdebug("addref rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));
5554
refcount_inc(&rm->m_refcount);
5655
}
5756
EXPORT_SYMBOL_GPL(rds_message_addref);
5857

59-
static inline bool skb_zcookie_add(struct sk_buff *skb, u32 cookie)
58+
static inline bool rds_zcookie_add(struct rds_msg_zcopy_info *info, u32 cookie)
6059
{
61-
struct rds_zcopy_cookies *ck = (struct rds_zcopy_cookies *)skb->cb;
60+
struct rds_zcopy_cookies *ck = &info->zcookies;
6261
int ncookies = ck->num;
6362

6463
if (ncookies == RDS_MAX_ZCOOKIES)
@@ -68,38 +67,61 @@ static inline bool skb_zcookie_add(struct sk_buff *skb, u32 cookie)
6867
return true;
6968
}
7069

70+
struct rds_msg_zcopy_info *rds_info_from_znotifier(struct rds_znotifier *znotif)
71+
{
72+
return container_of(znotif, struct rds_msg_zcopy_info, znotif);
73+
}
74+
75+
void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *q)
76+
{
77+
unsigned long flags;
78+
LIST_HEAD(copy);
79+
struct rds_msg_zcopy_info *info, *tmp;
80+
81+
spin_lock_irqsave(&q->lock, flags);
82+
list_splice(&q->zcookie_head, &copy);
83+
INIT_LIST_HEAD(&q->zcookie_head);
84+
spin_unlock_irqrestore(&q->lock, flags);
85+
86+
list_for_each_entry_safe(info, tmp, &copy, rs_zcookie_next) {
87+
list_del(&info->rs_zcookie_next);
88+
kfree(info);
89+
}
90+
}
91+
7192
static void rds_rm_zerocopy_callback(struct rds_sock *rs,
7293
struct rds_znotifier *znotif)
7394
{
74-
struct sk_buff *skb, *tail;
75-
unsigned long flags;
76-
struct sk_buff_head *q;
95+
struct rds_msg_zcopy_info *info;
96+
struct rds_msg_zcopy_queue *q;
7797
u32 cookie = znotif->z_cookie;
7898
struct rds_zcopy_cookies *ck;
99+
struct list_head *head;
100+
unsigned long flags;
79101

102+
mm_unaccount_pinned_pages(&znotif->z_mmp);
80103
q = &rs->rs_zcookie_queue;
81104
spin_lock_irqsave(&q->lock, flags);
82-
tail = skb_peek_tail(q);
83-
84-
if (tail && skb_zcookie_add(tail, cookie)) {
85-
spin_unlock_irqrestore(&q->lock, flags);
86-
mm_unaccount_pinned_pages(&znotif->z_mmp);
87-
consume_skb(rds_skb_from_znotifier(znotif));
88-
/* caller invokes rds_wake_sk_sleep() */
89-
return;
105+
head = &q->zcookie_head;
106+
if (!list_empty(head)) {
107+
info = list_entry(head, struct rds_msg_zcopy_info,
108+
rs_zcookie_next);
109+
if (info && rds_zcookie_add(info, cookie)) {
110+
spin_unlock_irqrestore(&q->lock, flags);
111+
kfree(rds_info_from_znotifier(znotif));
112+
/* caller invokes rds_wake_sk_sleep() */
113+
return;
114+
}
90115
}
91116

92-
skb = rds_skb_from_znotifier(znotif);
93-
ck = (struct rds_zcopy_cookies *)skb->cb;
117+
info = rds_info_from_znotifier(znotif);
118+
ck = &info->zcookies;
94119
memset(ck, 0, sizeof(*ck));
95-
WARN_ON(!skb_zcookie_add(skb, cookie));
96-
97-
__skb_queue_tail(q, skb);
120+
WARN_ON(!rds_zcookie_add(info, cookie));
121+
list_add_tail(&q->zcookie_head, &info->rs_zcookie_next);
98122

99123
spin_unlock_irqrestore(&q->lock, flags);
100124
/* caller invokes rds_wake_sk_sleep() */
101-
102-
mm_unaccount_pinned_pages(&znotif->z_mmp);
103125
}
104126

105127
/*
@@ -333,14 +355,14 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in
333355
return rm;
334356
}
335357

336-
int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
337-
bool zcopy)
358+
int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *from)
338359
{
339-
unsigned long to_copy, nbytes;
340360
unsigned long sg_off;
341361
struct scatterlist *sg;
342362
int ret = 0;
343363
int length = iov_iter_count(from);
364+
int total_copied = 0;
365+
struct rds_msg_zcopy_info *info;
344366

345367
rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));
346368

@@ -350,54 +372,65 @@ int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
350372
sg = rm->data.op_sg;
351373
sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */
352374

353-
if (zcopy) {
354-
int total_copied = 0;
355-
struct sk_buff *skb;
356-
357-
skb = alloc_skb(0, GFP_KERNEL);
358-
if (!skb)
359-
return -ENOMEM;
360-
BUILD_BUG_ON(sizeof(skb->cb) <
361-
max_t(int, sizeof(struct rds_znotifier),
362-
sizeof(struct rds_zcopy_cookies)));
363-
rm->data.op_mmp_znotifier = RDS_ZCOPY_SKB(skb);
364-
if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp,
365-
length)) {
366-
ret = -ENOMEM;
375+
info = kzalloc(sizeof(*info), GFP_KERNEL);
376+
if (!info)
377+
return -ENOMEM;
378+
INIT_LIST_HEAD(&info->rs_zcookie_next);
379+
rm->data.op_mmp_znotifier = &info->znotif;
380+
if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp,
381+
length)) {
382+
ret = -ENOMEM;
383+
goto err;
384+
}
385+
while (iov_iter_count(from)) {
386+
struct page *pages;
387+
size_t start;
388+
ssize_t copied;
389+
390+
copied = iov_iter_get_pages(from, &pages, PAGE_SIZE,
391+
1, &start);
392+
if (copied < 0) {
393+
struct mmpin *mmp;
394+
int i;
395+
396+
for (i = 0; i < rm->data.op_nents; i++)
397+
put_page(sg_page(&rm->data.op_sg[i]));
398+
mmp = &rm->data.op_mmp_znotifier->z_mmp;
399+
mm_unaccount_pinned_pages(mmp);
400+
ret = -EFAULT;
367401
goto err;
368402
}
369-
while (iov_iter_count(from)) {
370-
struct page *pages;
371-
size_t start;
372-
ssize_t copied;
373-
374-
copied = iov_iter_get_pages(from, &pages, PAGE_SIZE,
375-
1, &start);
376-
if (copied < 0) {
377-
struct mmpin *mmp;
378-
int i;
379-
380-
for (i = 0; i < rm->data.op_nents; i++)
381-
put_page(sg_page(&rm->data.op_sg[i]));
382-
mmp = &rm->data.op_mmp_znotifier->z_mmp;
383-
mm_unaccount_pinned_pages(mmp);
384-
ret = -EFAULT;
385-
goto err;
386-
}
387-
total_copied += copied;
388-
iov_iter_advance(from, copied);
389-
length -= copied;
390-
sg_set_page(sg, pages, copied, start);
391-
rm->data.op_nents++;
392-
sg++;
393-
}
394-
WARN_ON_ONCE(length != 0);
395-
return ret;
403+
total_copied += copied;
404+
iov_iter_advance(from, copied);
405+
length -= copied;
406+
sg_set_page(sg, pages, copied, start);
407+
rm->data.op_nents++;
408+
sg++;
409+
}
410+
WARN_ON_ONCE(length != 0);
411+
return ret;
396412
err:
397-
consume_skb(skb);
398-
rm->data.op_mmp_znotifier = NULL;
399-
return ret;
400-
} /* zcopy */
413+
kfree(info);
414+
rm->data.op_mmp_znotifier = NULL;
415+
return ret;
416+
}
417+
418+
int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
419+
bool zcopy)
420+
{
421+
unsigned long to_copy, nbytes;
422+
unsigned long sg_off;
423+
struct scatterlist *sg;
424+
int ret = 0;
425+
426+
rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));
427+
428+
/* now allocate and copy in the data payload. */
429+
sg = rm->data.op_sg;
430+
sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */
431+
432+
if (zcopy)
433+
return rds_message_zcopy_from_user(rm, from);
401434

402435
while (iov_iter_count(from)) {
403436
if (!sg_page(sg)) {

‎net/rds/rds.h

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -357,16 +357,27 @@ static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
357357
#define RDS_MSG_FLUSH 8
358358

359359
struct rds_znotifier {
360-
struct list_head z_list;
361360
struct mmpin z_mmp;
362361
u32 z_cookie;
363362
};
364363

365-
#define RDS_ZCOPY_SKB(__skb) ((struct rds_znotifier *)&((__skb)->cb[0]))
364+
struct rds_msg_zcopy_info {
365+
struct list_head rs_zcookie_next;
366+
union {
367+
struct rds_znotifier znotif;
368+
struct rds_zcopy_cookies zcookies;
369+
};
370+
};
366371

367-
static inline struct sk_buff *rds_skb_from_znotifier(struct rds_znotifier *z)
372+
struct rds_msg_zcopy_queue {
373+
struct list_head zcookie_head;
374+
spinlock_t lock; /* protects zcookie_head queue */
375+
};
376+
377+
static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
368378
{
369-
return container_of((void *)z, struct sk_buff, cb);
379+
spin_lock_init(&q->lock);
380+
INIT_LIST_HEAD(&q->zcookie_head);
370381
}
371382

372383
struct rds_message {
@@ -603,8 +614,7 @@ struct rds_sock {
603614
/* Socket receive path trace points*/
604615
u8 rs_rx_traces;
605616
u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
606-
607-
struct sk_buff_head rs_zcookie_queue;
617+
struct rds_msg_zcopy_queue rs_zcookie_queue;
608618
};
609619

610620
static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
@@ -803,6 +813,7 @@ void rds_message_addref(struct rds_message *rm);
803813
void rds_message_put(struct rds_message *rm);
804814
void rds_message_wait(struct rds_message *rm);
805815
void rds_message_unmapped(struct rds_message *rm);
816+
void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
806817

807818
static inline void rds_message_make_checksum(struct rds_header *hdr)
808819
{

‎net/rds/recv.c

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -579,9 +579,10 @@ static int rds_cmsg_recv(struct rds_incoming *inc, struct msghdr *msg,
579579

580580
static bool rds_recvmsg_zcookie(struct rds_sock *rs, struct msghdr *msg)
581581
{
582-
struct sk_buff *skb;
583-
struct sk_buff_head *q = &rs->rs_zcookie_queue;
582+
struct rds_msg_zcopy_queue *q = &rs->rs_zcookie_queue;
583+
struct rds_msg_zcopy_info *info = NULL;
584584
struct rds_zcopy_cookies *done;
585+
unsigned long flags;
585586

586587
if (!msg->msg_control)
587588
return false;
@@ -590,16 +591,24 @@ static bool rds_recvmsg_zcookie(struct rds_sock *rs, struct msghdr *msg)
590591
msg->msg_controllen < CMSG_SPACE(sizeof(*done)))
591592
return false;
592593

593-
skb = skb_dequeue(q);
594-
if (!skb)
594+
spin_lock_irqsave(&q->lock, flags);
595+
if (!list_empty(&q->zcookie_head)) {
596+
info = list_entry(q->zcookie_head.next,
597+
struct rds_msg_zcopy_info, rs_zcookie_next);
598+
list_del(&info->rs_zcookie_next);
599+
}
600+
spin_unlock_irqrestore(&q->lock, flags);
601+
if (!info)
595602
return false;
596-
done = (struct rds_zcopy_cookies *)skb->cb;
603+
done = &info->zcookies;
597604
if (put_cmsg(msg, SOL_RDS, RDS_CMSG_ZCOPY_COMPLETION, sizeof(*done),
598605
done)) {
599-
skb_queue_head(q, skb);
606+
spin_lock_irqsave(&q->lock, flags);
607+
list_add(&info->rs_zcookie_next, &q->zcookie_head);
608+
spin_unlock_irqrestore(&q->lock, flags);
600609
return false;
601610
}
602-
consume_skb(skb);
611+
kfree(info);
603612
return true;
604613
}
605614

0 commit comments

Comments
 (0)