Skip to content

Commit 7439d68

Browse files
Paolo Abenikuba-moo
Paolo Abeni
authored and
kuba-moo
committed
mptcp: avoid a few atomic ops in the rx path
Extending the data_lock scope in mptcp_incoming_option we can use that to protect both snd_una and wnd_end. In the typical case, we will have a single atomic op instead of 2 Acked-by: Florian Westphal <[email protected]> Signed-off-by: Paolo Abeni <[email protected]> Reviewed-by: Mat Martineau <[email protected]> Signed-off-by: Jakub Kicinski <[email protected]>
1 parent 724cfd2 commit 7439d68

File tree

4 files changed

+34
-43
lines changed

4 files changed

+34
-43
lines changed

‎net/mptcp/mptcp_diag.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ static void mptcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r,
140140
info->mptcpi_flags = flags;
141141
info->mptcpi_token = READ_ONCE(msk->token);
142142
info->mptcpi_write_seq = READ_ONCE(msk->write_seq);
143-
info->mptcpi_snd_una = atomic64_read(&msk->snd_una);
143+
info->mptcpi_snd_una = READ_ONCE(msk->snd_una);
144144
info->mptcpi_rcv_nxt = READ_ONCE(msk->ack_seq);
145145
unlock_sock_fast(sk, slow);
146146
}

‎net/mptcp/options.c

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -833,15 +833,17 @@ static void ack_update_msk(struct mptcp_sock *msk,
833833
const struct sock *ssk,
834834
struct mptcp_options_received *mp_opt)
835835
{
836-
u64 new_snd_una, snd_una, old_snd_una = atomic64_read(&msk->snd_una);
837-
u64 new_wnd_end, wnd_end, old_wnd_end = atomic64_read(&msk->wnd_end);
838-
u64 snd_nxt = READ_ONCE(msk->snd_nxt);
836+
u64 new_wnd_end, new_snd_una, snd_nxt = READ_ONCE(msk->snd_nxt);
839837
struct sock *sk = (struct sock *)msk;
838+
u64 old_snd_una;
839+
840+
mptcp_data_lock(sk);
840841

841842
/* avoid ack expansion on update conflict, to reduce the risk of
842843
* wrongly expanding to a future ack sequence number, which is way
843844
* more dangerous than missing an ack
844845
*/
846+
old_snd_una = msk->snd_una;
845847
new_snd_una = expand_ack(old_snd_una, mp_opt->data_ack, mp_opt->ack64);
846848

847849
/* ACK for data not even sent yet? Ignore. */
@@ -850,26 +852,17 @@ static void ack_update_msk(struct mptcp_sock *msk,
850852

851853
new_wnd_end = new_snd_una + tcp_sk(ssk)->snd_wnd;
852854

853-
while (after64(new_wnd_end, old_wnd_end)) {
854-
wnd_end = old_wnd_end;
855-
old_wnd_end = atomic64_cmpxchg(&msk->wnd_end, wnd_end,
856-
new_wnd_end);
857-
if (old_wnd_end == wnd_end) {
858-
if (mptcp_send_head(sk))
859-
mptcp_schedule_work(sk);
860-
break;
861-
}
855+
if (after64(new_wnd_end, msk->wnd_end)) {
856+
msk->wnd_end = new_wnd_end;
857+
if (mptcp_send_head(sk))
858+
mptcp_schedule_work(sk);
862859
}
863860

864-
while (after64(new_snd_una, old_snd_una)) {
865-
snd_una = old_snd_una;
866-
old_snd_una = atomic64_cmpxchg(&msk->snd_una, snd_una,
867-
new_snd_una);
868-
if (old_snd_una == snd_una) {
869-
mptcp_data_acked(sk);
870-
break;
871-
}
861+
if (after64(new_snd_una, old_snd_una)) {
862+
msk->snd_una = new_snd_una;
863+
__mptcp_data_acked(sk);
872864
}
865+
mptcp_data_unlock(sk);
873866
}
874867

875868
bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit)

‎net/mptcp/protocol.c

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
6060
/* Returns end sequence number of the receiver's advertised window */
6161
static u64 mptcp_wnd_end(const struct mptcp_sock *msk)
6262
{
63-
return atomic64_read(&msk->wnd_end);
63+
return READ_ONCE(msk->wnd_end);
6464
}
6565

6666
static bool mptcp_is_tcpsk(struct sock *sk)
@@ -358,7 +358,7 @@ static void mptcp_check_data_fin_ack(struct sock *sk)
358358
/* Look for an acknowledged DATA_FIN */
359359
if (((1 << sk->sk_state) &
360360
(TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) &&
361-
msk->write_seq == atomic64_read(&msk->snd_una)) {
361+
msk->write_seq == READ_ONCE(msk->snd_una)) {
362362
mptcp_stop_timer(sk);
363363

364364
WRITE_ONCE(msk->snd_data_fin_enable, 0);
@@ -764,7 +764,7 @@ bool mptcp_schedule_work(struct sock *sk)
764764
return false;
765765
}
766766

767-
void mptcp_data_acked(struct sock *sk)
767+
void __mptcp_data_acked(struct sock *sk)
768768
{
769769
mptcp_reset_timer(sk);
770770

@@ -997,11 +997,11 @@ static void mptcp_clean_una(struct sock *sk)
997997
* plain TCP
998998
*/
999999
if (__mptcp_check_fallback(msk))
1000-
atomic64_set(&msk->snd_una, msk->snd_nxt);
1000+
msk->snd_una = READ_ONCE(msk->snd_nxt);
10011001

1002-
mptcp_data_lock(sk);
1003-
snd_una = atomic64_read(&msk->snd_una);
10041002

1003+
mptcp_data_lock(sk);
1004+
snd_una = msk->snd_una;
10051005
list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {
10061006
if (after64(dfrag->data_seq + dfrag->data_len, snd_una))
10071007
break;
@@ -1282,10 +1282,12 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
12821282
/* Zero window and all data acked? Probe. */
12831283
avail_size = mptcp_check_allowed_size(msk, data_seq, avail_size);
12841284
if (avail_size == 0) {
1285-
if (skb || atomic64_read(&msk->snd_una) != msk->snd_nxt)
1285+
u64 snd_una = READ_ONCE(msk->snd_una);
1286+
1287+
if (skb || snd_una != msk->snd_nxt)
12861288
return 0;
12871289
zero_window_probe = true;
1288-
data_seq = atomic64_read(&msk->snd_una) - 1;
1290+
data_seq = snd_una - 1;
12891291
avail_size = 1;
12901292
}
12911293

@@ -1994,12 +1996,8 @@ static void mptcp_retransmit_handler(struct sock *sk)
19941996
{
19951997
struct mptcp_sock *msk = mptcp_sk(sk);
19961998

1997-
if (atomic64_read(&msk->snd_una) == READ_ONCE(msk->snd_nxt)) {
1998-
mptcp_stop_timer(sk);
1999-
} else {
2000-
set_bit(MPTCP_WORK_RTX, &msk->flags);
2001-
mptcp_schedule_work(sk);
2002-
}
1999+
set_bit(MPTCP_WORK_RTX, &msk->flags);
2000+
mptcp_schedule_work(sk);
20032001
}
20042002

20052003
static void mptcp_retransmit_timer(struct timer_list *t)
@@ -2621,8 +2619,8 @@ struct sock *mptcp_sk_clone(const struct sock *sk,
26212619

26222620
msk->write_seq = subflow_req->idsn + 1;
26232621
msk->snd_nxt = msk->write_seq;
2624-
atomic64_set(&msk->snd_una, msk->write_seq);
2625-
atomic64_set(&msk->wnd_end, msk->snd_nxt + req->rsk_rcv_wnd);
2622+
msk->snd_una = msk->write_seq;
2623+
msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd;
26262624

26272625
if (mp_opt->mp_capable) {
26282626
msk->can_ack = true;
@@ -2658,7 +2656,7 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)
26582656
if (msk->rcvq_space.space == 0)
26592657
msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT;
26602658

2661-
atomic64_set(&msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd);
2659+
WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd);
26622660
}
26632661

26642662
static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
@@ -2918,7 +2916,7 @@ void mptcp_finish_connect(struct sock *ssk)
29182916
WRITE_ONCE(msk->ack_seq, ack_seq);
29192917
WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
29202918
WRITE_ONCE(msk->can_ack, 1);
2921-
atomic64_set(&msk->snd_una, msk->write_seq);
2919+
WRITE_ONCE(msk->snd_una, msk->write_seq);
29222920

29232921
mptcp_pm_new_connection(msk, 0);
29242922

‎net/mptcp/protocol.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,8 +222,8 @@ struct mptcp_sock {
222222
struct sock *last_snd;
223223
int snd_burst;
224224
int old_wspace;
225-
atomic64_t snd_una;
226-
atomic64_t wnd_end;
225+
u64 snd_una;
226+
u64 wnd_end;
227227
unsigned long timer_ival;
228228
u32 token;
229229
int rmem_pending;
@@ -321,7 +321,7 @@ static inline struct mptcp_data_frag *mptcp_rtx_tail(const struct sock *sk)
321321
{
322322
struct mptcp_sock *msk = mptcp_sk(sk);
323323

324-
if (!before64(msk->snd_nxt, atomic64_read(&msk->snd_una)))
324+
if (!before64(msk->snd_nxt, READ_ONCE(msk->snd_una)))
325325
return NULL;
326326

327327
return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list);
@@ -495,7 +495,7 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk);
495495
void mptcp_data_ready(struct sock *sk, struct sock *ssk);
496496
bool mptcp_finish_join(struct sock *sk);
497497
bool mptcp_schedule_work(struct sock *sk);
498-
void mptcp_data_acked(struct sock *sk);
498+
void __mptcp_data_acked(struct sock *sk);
499499
void mptcp_subflow_eof(struct sock *sk);
500500
bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit);
501501
void __mptcp_flush_join_list(struct mptcp_sock *msk);

0 commit comments

Comments
 (0)