Skip to content

Commit ac69269

Browse files
Julian Anastasovummakynes
Julian Anastasov
authored and
ummakynes
committed
ipvs: do not disable bh for long time
We used a global BH disable in LOCAL_OUT hook. Add _bh suffix to all places that need it and remove the disabling from LOCAL_OUT and sync code. Functions like ip_defrag need protection from BH, so add it. As for nf_nat_mangle_tcp_packet, it needs RCU lock. Signed-off-by: Julian Anastasov <[email protected]> Signed-off-by: Simon Horman <[email protected]>
1 parent ceec4c3 commit ac69269

File tree

12 files changed

+64
-87
lines changed

12 files changed

+64
-87
lines changed

‎net/netfilter/ipvs/ip_vs_app.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,14 +352,14 @@ static inline void vs_seq_update(struct ip_vs_conn *cp, struct ip_vs_seq *vseq,
352352
unsigned int flag, __u32 seq, int diff)
353353
{
354354
/* spinlock is to keep updating cp->flags atomic */
355-
spin_lock(&cp->lock);
355+
spin_lock_bh(&cp->lock);
356356
if (!(cp->flags & flag) || after(seq, vseq->init_seq)) {
357357
vseq->previous_delta = vseq->delta;
358358
vseq->delta += diff;
359359
vseq->init_seq = seq;
360360
cp->flags |= flag;
361361
}
362-
spin_unlock(&cp->lock);
362+
spin_unlock_bh(&cp->lock);
363363
}
364364

365365
static inline int app_tcp_pkt_out(struct ip_vs_conn *cp, struct sk_buff *skb,

‎net/netfilter/ipvs/ip_vs_conn.c

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,14 @@ struct ip_vs_aligned_lock
8686
static struct ip_vs_aligned_lock
8787
__ip_vs_conntbl_lock_array[CT_LOCKARRAY_SIZE] __cacheline_aligned;
8888

89-
static inline void ct_write_lock(unsigned int key)
89+
static inline void ct_write_lock_bh(unsigned int key)
9090
{
91-
spin_lock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
91+
spin_lock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
9292
}
9393

94-
static inline void ct_write_unlock(unsigned int key)
94+
static inline void ct_write_unlock_bh(unsigned int key)
9595
{
96-
spin_unlock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
96+
spin_unlock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
9797
}
9898

9999

@@ -167,7 +167,7 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp)
167167
/* Hash by protocol, client address and port */
168168
hash = ip_vs_conn_hashkey_conn(cp);
169169

170-
ct_write_lock(hash);
170+
ct_write_lock_bh(hash);
171171
spin_lock(&cp->lock);
172172

173173
if (!(cp->flags & IP_VS_CONN_F_HASHED)) {
@@ -182,7 +182,7 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp)
182182
}
183183

184184
spin_unlock(&cp->lock);
185-
ct_write_unlock(hash);
185+
ct_write_unlock_bh(hash);
186186

187187
return ret;
188188
}
@@ -200,7 +200,7 @@ static inline int ip_vs_conn_unhash(struct ip_vs_conn *cp)
200200
/* unhash it and decrease its reference counter */
201201
hash = ip_vs_conn_hashkey_conn(cp);
202202

203-
ct_write_lock(hash);
203+
ct_write_lock_bh(hash);
204204
spin_lock(&cp->lock);
205205

206206
if (cp->flags & IP_VS_CONN_F_HASHED) {
@@ -212,7 +212,7 @@ static inline int ip_vs_conn_unhash(struct ip_vs_conn *cp)
212212
ret = 0;
213213

214214
spin_unlock(&cp->lock);
215-
ct_write_unlock(hash);
215+
ct_write_unlock_bh(hash);
216216

217217
return ret;
218218
}
@@ -227,7 +227,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp)
227227

228228
hash = ip_vs_conn_hashkey_conn(cp);
229229

230-
ct_write_lock(hash);
230+
ct_write_lock_bh(hash);
231231
spin_lock(&cp->lock);
232232

233233
if (cp->flags & IP_VS_CONN_F_HASHED) {
@@ -242,7 +242,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp)
242242
ret = atomic_read(&cp->refcnt) ? false : true;
243243

244244
spin_unlock(&cp->lock);
245-
ct_write_unlock(hash);
245+
ct_write_unlock_bh(hash);
246246

247247
return ret;
248248
}
@@ -462,13 +462,13 @@ void ip_vs_conn_put(struct ip_vs_conn *cp)
462462
void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
463463
{
464464
if (ip_vs_conn_unhash(cp)) {
465-
spin_lock(&cp->lock);
465+
spin_lock_bh(&cp->lock);
466466
if (cp->flags & IP_VS_CONN_F_NO_CPORT) {
467467
atomic_dec(&ip_vs_conn_no_cport_cnt);
468468
cp->flags &= ~IP_VS_CONN_F_NO_CPORT;
469469
cp->cport = cport;
470470
}
471-
spin_unlock(&cp->lock);
471+
spin_unlock_bh(&cp->lock);
472472

473473
/* hash on new dport */
474474
ip_vs_conn_hash(cp);
@@ -622,9 +622,9 @@ void ip_vs_try_bind_dest(struct ip_vs_conn *cp)
622622
if (dest) {
623623
struct ip_vs_proto_data *pd;
624624

625-
spin_lock(&cp->lock);
625+
spin_lock_bh(&cp->lock);
626626
if (cp->dest) {
627-
spin_unlock(&cp->lock);
627+
spin_unlock_bh(&cp->lock);
628628
rcu_read_unlock();
629629
return;
630630
}
@@ -635,7 +635,7 @@ void ip_vs_try_bind_dest(struct ip_vs_conn *cp)
635635
ip_vs_unbind_app(cp);
636636

637637
ip_vs_bind_dest(cp, dest);
638-
spin_unlock(&cp->lock);
638+
spin_unlock_bh(&cp->lock);
639639

640640
/* Update its packet transmitter */
641641
cp->packet_xmit = NULL;

‎net/netfilter/ipvs/ip_vs_core.c

Lines changed: 8 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -638,8 +638,11 @@ static inline enum ip_defrag_users ip_vs_defrag_user(unsigned int hooknum)
638638

639639
static inline int ip_vs_gather_frags(struct sk_buff *skb, u_int32_t user)
640640
{
641-
int err = ip_defrag(skb, user);
641+
int err;
642642

643+
local_bh_disable();
644+
err = ip_defrag(skb, user);
645+
local_bh_enable();
643646
if (!err)
644647
ip_send_check(ip_hdr(skb));
645648

@@ -1217,13 +1220,7 @@ ip_vs_local_reply4(unsigned int hooknum, struct sk_buff *skb,
12171220
const struct net_device *in, const struct net_device *out,
12181221
int (*okfn)(struct sk_buff *))
12191222
{
1220-
unsigned int verdict;
1221-
1222-
/* Disable BH in LOCAL_OUT until all places are fixed */
1223-
local_bh_disable();
1224-
verdict = ip_vs_out(hooknum, skb, AF_INET);
1225-
local_bh_enable();
1226-
return verdict;
1223+
return ip_vs_out(hooknum, skb, AF_INET);
12271224
}
12281225

12291226
#ifdef CONFIG_IP_VS_IPV6
@@ -1250,13 +1247,7 @@ ip_vs_local_reply6(unsigned int hooknum, struct sk_buff *skb,
12501247
const struct net_device *in, const struct net_device *out,
12511248
int (*okfn)(struct sk_buff *))
12521249
{
1253-
unsigned int verdict;
1254-
1255-
/* Disable BH in LOCAL_OUT until all places are fixed */
1256-
local_bh_disable();
1257-
verdict = ip_vs_out(hooknum, skb, AF_INET6);
1258-
local_bh_enable();
1259-
return verdict;
1250+
return ip_vs_out(hooknum, skb, AF_INET6);
12601251
}
12611252

12621253
#endif
@@ -1714,13 +1705,7 @@ ip_vs_local_request4(unsigned int hooknum, struct sk_buff *skb,
17141705
const struct net_device *in, const struct net_device *out,
17151706
int (*okfn)(struct sk_buff *))
17161707
{
1717-
unsigned int verdict;
1718-
1719-
/* Disable BH in LOCAL_OUT until all places are fixed */
1720-
local_bh_disable();
1721-
verdict = ip_vs_in(hooknum, skb, AF_INET);
1722-
local_bh_enable();
1723-
return verdict;
1708+
return ip_vs_in(hooknum, skb, AF_INET);
17241709
}
17251710

17261711
#ifdef CONFIG_IP_VS_IPV6
@@ -1779,13 +1764,7 @@ ip_vs_local_request6(unsigned int hooknum, struct sk_buff *skb,
17791764
const struct net_device *in, const struct net_device *out,
17801765
int (*okfn)(struct sk_buff *))
17811766
{
1782-
unsigned int verdict;
1783-
1784-
/* Disable BH in LOCAL_OUT until all places are fixed */
1785-
local_bh_disable();
1786-
verdict = ip_vs_in(hooknum, skb, AF_INET6);
1787-
local_bh_enable();
1788-
return verdict;
1767+
return ip_vs_in(hooknum, skb, AF_INET6);
17891768
}
17901769

17911770
#endif

‎net/netfilter/ipvs/ip_vs_ftp.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,10 +267,12 @@ static int ip_vs_ftp_out(struct ip_vs_app *app, struct ip_vs_conn *cp,
267267
* hopefully it will succeed on the retransmitted
268268
* packet.
269269
*/
270+
rcu_read_lock();
270271
ret = nf_nat_mangle_tcp_packet(skb, ct, ctinfo,
271272
iph->ihl * 4,
272273
start-data, end-start,
273274
buf, buf_len);
275+
rcu_read_unlock();
274276
if (ret) {
275277
ip_vs_nfct_expect_related(skb, ct, n_cp,
276278
IPPROTO_TCP, 0, 0);

‎net/netfilter/ipvs/ip_vs_lblc.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -527,10 +527,10 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
527527
}
528528

529529
/* If we fail to create a cache entry, we'll just use the valid dest */
530-
spin_lock(&svc->sched_lock);
530+
spin_lock_bh(&svc->sched_lock);
531531
if (!tbl->dead)
532532
ip_vs_lblc_new(tbl, &iph.daddr, dest);
533-
spin_unlock(&svc->sched_lock);
533+
spin_unlock_bh(&svc->sched_lock);
534534

535535
out:
536536
IP_VS_DBG_BUF(6, "LBLC: destination IP address %s --> server %s:%d\n",

‎net/netfilter/ipvs/ip_vs_lblcr.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -678,15 +678,15 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
678678
if (atomic_read(&en->set.size) > 1 &&
679679
time_after(jiffies, en->set.lastmod +
680680
sysctl_lblcr_expiration(svc))) {
681-
spin_lock(&svc->sched_lock);
681+
spin_lock_bh(&svc->sched_lock);
682682
if (atomic_read(&en->set.size) > 1) {
683683
struct ip_vs_dest *m;
684684

685685
m = ip_vs_dest_set_max(&en->set);
686686
if (m)
687687
ip_vs_dest_set_erase(&en->set, m);
688688
}
689-
spin_unlock(&svc->sched_lock);
689+
spin_unlock_bh(&svc->sched_lock);
690690
}
691691

692692
/* If the destination is not overloaded, use it */
@@ -701,10 +701,10 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
701701
}
702702

703703
/* Update our cache entry */
704-
spin_lock(&svc->sched_lock);
704+
spin_lock_bh(&svc->sched_lock);
705705
if (!tbl->dead)
706706
ip_vs_dest_set_insert(&en->set, dest, true);
707-
spin_unlock(&svc->sched_lock);
707+
spin_unlock_bh(&svc->sched_lock);
708708
goto out;
709709
}
710710

@@ -716,10 +716,10 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
716716
}
717717

718718
/* If we fail to create a cache entry, we'll just use the valid dest */
719-
spin_lock(&svc->sched_lock);
719+
spin_lock_bh(&svc->sched_lock);
720720
if (!tbl->dead)
721721
ip_vs_lblcr_new(tbl, &iph.daddr, dest);
722-
spin_unlock(&svc->sched_lock);
722+
spin_unlock_bh(&svc->sched_lock);
723723

724724
out:
725725
IP_VS_DBG_BUF(6, "LBLCR: destination IP address %s --> server %s:%d\n",

‎net/netfilter/ipvs/ip_vs_proto_sctp.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -994,9 +994,9 @@ static void
994994
sctp_state_transition(struct ip_vs_conn *cp, int direction,
995995
const struct sk_buff *skb, struct ip_vs_proto_data *pd)
996996
{
997-
spin_lock(&cp->lock);
997+
spin_lock_bh(&cp->lock);
998998
set_sctp_state(pd, cp, direction, skb);
999-
spin_unlock(&cp->lock);
999+
spin_unlock_bh(&cp->lock);
10001000
}
10011001

10021002
static inline __u16 sctp_app_hashkey(__be16 port)

‎net/netfilter/ipvs/ip_vs_proto_tcp.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -557,9 +557,9 @@ tcp_state_transition(struct ip_vs_conn *cp, int direction,
557557
if (th == NULL)
558558
return;
559559

560-
spin_lock(&cp->lock);
560+
spin_lock_bh(&cp->lock);
561561
set_tcp_state(pd, cp, direction, th);
562-
spin_unlock(&cp->lock);
562+
spin_unlock_bh(&cp->lock);
563563
}
564564

565565
static inline __u16 tcp_app_hashkey(__be16 port)
@@ -655,11 +655,11 @@ void ip_vs_tcp_conn_listen(struct net *net, struct ip_vs_conn *cp)
655655
{
656656
struct ip_vs_proto_data *pd = ip_vs_proto_data_get(net, IPPROTO_TCP);
657657

658-
spin_lock(&cp->lock);
658+
spin_lock_bh(&cp->lock);
659659
cp->state = IP_VS_TCP_S_LISTEN;
660660
cp->timeout = (pd ? pd->timeout_table[IP_VS_TCP_S_LISTEN]
661661
: tcp_timeouts[IP_VS_TCP_S_LISTEN]);
662-
spin_unlock(&cp->lock);
662+
spin_unlock_bh(&cp->lock);
663663
}
664664

665665
/* ---------------------------------------------

‎net/netfilter/ipvs/ip_vs_rr.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ ip_vs_rr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
6363

6464
IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);
6565

66-
spin_lock(&svc->sched_lock);
66+
spin_lock_bh(&svc->sched_lock);
6767
p = (struct list_head *) svc->sched_data;
6868
last = dest = list_entry(p, struct ip_vs_dest, n_list);
6969

@@ -85,13 +85,13 @@ ip_vs_rr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
8585
} while (pass < 2 && p != &svc->destinations);
8686

8787
stop:
88-
spin_unlock(&svc->sched_lock);
88+
spin_unlock_bh(&svc->sched_lock);
8989
ip_vs_scheduler_err(svc, "no destination available");
9090
return NULL;
9191

9292
out:
9393
svc->sched_data = &dest->n_list;
94-
spin_unlock(&svc->sched_lock);
94+
spin_unlock_bh(&svc->sched_lock);
9595
IP_VS_DBG_BUF(6, "RR: server %s:%u "
9696
"activeconns %d refcnt %d weight %d\n",
9797
IP_VS_DBG_ADDR(svc->af, &dest->addr), ntohs(dest->port),

0 commit comments

Comments
 (0)