Skip to content

Commit fdf9ef8

Browse files
davem330davem330
committed
Merge branch 'listen_refactor_part_13'
Eric Dumazet says: ==================== inet: tcp listener refactoring, part 13 inet_hash functions are in a bad state : Too much IPv6/IPv4 copy/pasting. Lets refactor a bit. Idea is that we do not want to have an equivalent of inet_csk(sk)->icsk_af_ops for request socks in order to be able to use the right variant. In this patch series, I started to let IPv6/IPv4 converge to common helpers. Idea is to use ipv6_addr_set_v4mapped() even for AF_INET sockets, so that we can test if (sk->sk_family == AF_INET6 && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) to tell if we deal with an IPv6 socket, or IPv4 one, at least in slow paths. Ideally, we could save 8 bytes per struct sock_common, if we alias skc_daddr & skc_rcv_saddr to skc_v6_daddr[3]/skc_v6_rcv_saddr[3]. ==================== Signed-off-by: David S. Miller <[email protected]>
2 parents 8f6320d + 08d2cc3 commit fdf9ef8

File tree

17 files changed

+110
-174
lines changed

17 files changed

+110
-174
lines changed

‎include/linux/udp.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ static inline struct udphdr *inner_udp_hdr(const struct sk_buff *skb)
3434

3535
#define UDP_HTABLE_SIZE_MIN (CONFIG_BASE_SMALL ? 128 : 256)
3636

37-
static inline int udp_hashfn(struct net *net, unsigned num, unsigned mask)
37+
static inline u32 udp_hashfn(const struct net *net, u32 num, u32 mask)
3838
{
3939
return (num + net_hash_mix(net)) & mask;
4040
}

‎include/net/inet6_hashtables.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,6 @@ static inline unsigned int __inet6_ehashfn(const u32 lhash,
3838
return jhash_3words(lhash, fhash, ports, initval);
3939
}
4040

41-
int __inet6_hash(struct sock *sk, struct inet_timewait_sock *twp);
42-
4341
/*
4442
* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
4543
* we need not check it for TCP lookups anymore, thanks Alexey. -DaveM

‎include/net/inet_hashtables.h

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,8 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net,
221221
void inet_bind_bucket_destroy(struct kmem_cache *cachep,
222222
struct inet_bind_bucket *tb);
223223

224-
static inline int inet_bhashfn(struct net *net, const __u16 lport,
225-
const int bhash_size)
224+
static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,
225+
const u32 bhash_size)
226226
{
227227
return (lport + net_hash_mix(net)) & (bhash_size - 1);
228228
}
@@ -231,7 +231,7 @@ void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
231231
const unsigned short snum);
232232

233233
/* These can have wildcards, don't try too hard. */
234-
static inline int inet_lhashfn(struct net *net, const unsigned short num)
234+
static inline u32 inet_lhashfn(const struct net *net, const unsigned short num)
235235
{
236236
return (num + net_hash_mix(net)) & (INET_LHTABLE_SIZE - 1);
237237
}
@@ -249,6 +249,7 @@ void inet_put_port(struct sock *sk);
249249
void inet_hashinfo_init(struct inet_hashinfo *h);
250250

251251
int __inet_hash_nolisten(struct sock *sk, struct inet_timewait_sock *tw);
252+
int __inet_hash(struct sock *sk, struct inet_timewait_sock *tw);
252253
void inet_hash(struct sock *sk);
253254
void inet_unhash(struct sock *sk);
254255

@@ -383,13 +384,32 @@ static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo,
383384
iph->daddr, dport, inet_iif(skb));
384385
}
385386

387+
u32 sk_ehashfn(const struct sock *sk);
388+
u32 inet6_ehashfn(const struct net *net,
389+
const struct in6_addr *laddr, const u16 lport,
390+
const struct in6_addr *faddr, const __be16 fport);
391+
392+
static inline void sk_daddr_set(struct sock *sk, __be32 addr)
393+
{
394+
sk->sk_daddr = addr; /* alias of inet_daddr */
395+
#if IS_ENABLED(CONFIG_IPV6)
396+
ipv6_addr_set_v4mapped(addr, &sk->sk_v6_daddr);
397+
#endif
398+
}
399+
400+
static inline void sk_rcv_saddr_set(struct sock *sk, __be32 addr)
401+
{
402+
sk->sk_rcv_saddr = addr; /* alias of inet_rcv_saddr */
403+
#if IS_ENABLED(CONFIG_IPV6)
404+
ipv6_addr_set_v4mapped(addr, &sk->sk_v6_rcv_saddr);
405+
#endif
406+
}
407+
386408
int __inet_hash_connect(struct inet_timewait_death_row *death_row,
387409
struct sock *sk, u32 port_offset,
388410
int (*check_established)(struct inet_timewait_death_row *,
389411
struct sock *, __u16,
390-
struct inet_timewait_sock **),
391-
int (*hash)(struct sock *sk,
392-
struct inet_timewait_sock *twp));
412+
struct inet_timewait_sock **));
393413

394414
int inet_hash_connect(struct inet_timewait_death_row *death_row,
395415
struct sock *sk);

‎include/net/netns/hash.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@
55

66
struct net;
77

8-
static inline unsigned int net_hash_mix(struct net *net)
8+
static inline u32 net_hash_mix(const struct net *net)
99
{
1010
#ifdef CONFIG_NET_NS
1111
/*
1212
* shift this right to eliminate bits, that are
1313
* always zeroed
1414
*/
1515

16-
return (unsigned)(((unsigned long)net) >> L1_CACHE_SHIFT);
16+
return (u32)(((unsigned long)net) >> L1_CACHE_SHIFT);
1717
#else
1818
return 0;
1919
#endif

‎include/net/request_sock.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,11 @@ static inline struct request_sock *inet_reqsk(struct sock *sk)
9191
return (struct request_sock *)sk;
9292
}
9393

94+
static inline struct sock *req_to_sk(struct request_sock *req)
95+
{
96+
return (struct sock *)req;
97+
}
98+
9499
static inline void reqsk_free(struct request_sock *req)
95100
{
96101
/* temporary debugging */

‎net/dccp/ipv4.c

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,9 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
8989

9090
if (inet->inet_saddr == 0)
9191
inet->inet_saddr = fl4->saddr;
92-
inet->inet_rcv_saddr = inet->inet_saddr;
93-
92+
sk_rcv_saddr_set(sk, inet->inet_saddr);
9493
inet->inet_dport = usin->sin_port;
95-
inet->inet_daddr = daddr;
94+
sk_daddr_set(sk, daddr);
9695

9796
inet_csk(sk)->icsk_ext_hdr_len = 0;
9897
if (inet_opt)
@@ -408,8 +407,8 @@ struct sock *dccp_v4_request_recv_sock(struct sock *sk, struct sk_buff *skb,
408407

409408
newinet = inet_sk(newsk);
410409
ireq = inet_rsk(req);
411-
newinet->inet_daddr = ireq->ir_rmt_addr;
412-
newinet->inet_rcv_saddr = ireq->ir_loc_addr;
410+
sk_daddr_set(newsk, ireq->ir_rmt_addr);
411+
sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
413412
newinet->inet_saddr = ireq->ir_loc_addr;
414413
newinet->inet_opt = ireq->opt;
415414
ireq->opt = NULL;
@@ -639,8 +638,8 @@ int dccp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
639638
goto drop_and_free;
640639

641640
ireq = inet_rsk(req);
642-
ireq->ir_loc_addr = ip_hdr(skb)->daddr;
643-
ireq->ir_rmt_addr = ip_hdr(skb)->saddr;
641+
sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
642+
sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
644643
ireq->ireq_family = AF_INET;
645644
ireq->ir_iif = sk->sk_bound_dev_if;
646645

‎net/dccp/ipv6.c

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -40,19 +40,6 @@
4040
static const struct inet_connection_sock_af_ops dccp_ipv6_mapped;
4141
static const struct inet_connection_sock_af_ops dccp_ipv6_af_ops;
4242

43-
static void dccp_v6_hash(struct sock *sk)
44-
{
45-
if (sk->sk_state != DCCP_CLOSED) {
46-
if (inet_csk(sk)->icsk_af_ops == &dccp_ipv6_mapped) {
47-
inet_hash(sk);
48-
return;
49-
}
50-
local_bh_disable();
51-
__inet6_hash(sk, NULL);
52-
local_bh_enable();
53-
}
54-
}
55-
5643
/* add pseudo-header to DCCP checksum stored in skb->csum */
5744
static inline __sum16 dccp_v6_csum_finish(struct sk_buff *skb,
5845
const struct in6_addr *saddr,
@@ -470,11 +457,7 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk,
470457

471458
memcpy(newnp, np, sizeof(struct ipv6_pinfo));
472459

473-
ipv6_addr_set_v4mapped(newinet->inet_daddr, &newsk->sk_v6_daddr);
474-
475-
ipv6_addr_set_v4mapped(newinet->inet_saddr, &newnp->saddr);
476-
477-
newsk->sk_v6_rcv_saddr = newnp->saddr;
460+
newnp->saddr = newsk->sk_v6_rcv_saddr;
478461

479462
inet_csk(newsk)->icsk_af_ops = &dccp_ipv6_mapped;
480463
newsk->sk_backlog_rcv = dccp_v4_do_rcv;
@@ -592,7 +575,7 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk,
592575
dccp_done(newsk);
593576
goto out;
594577
}
595-
__inet6_hash(newsk, NULL);
578+
__inet_hash(newsk, NULL);
596579

597580
return newsk;
598581

@@ -917,9 +900,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
917900
sk->sk_backlog_rcv = dccp_v6_do_rcv;
918901
goto failure;
919902
}
920-
ipv6_addr_set_v4mapped(inet->inet_saddr, &np->saddr);
921-
ipv6_addr_set_v4mapped(inet->inet_rcv_saddr, &sk->sk_v6_rcv_saddr);
922-
903+
np->saddr = sk->sk_v6_rcv_saddr;
923904
return err;
924905
}
925906

@@ -1062,7 +1043,7 @@ static struct proto dccp_v6_prot = {
10621043
.sendmsg = dccp_sendmsg,
10631044
.recvmsg = dccp_recvmsg,
10641045
.backlog_rcv = dccp_v6_do_rcv,
1065-
.hash = dccp_v6_hash,
1046+
.hash = inet_hash,
10661047
.unhash = inet_unhash,
10671048
.accept = inet_csk_accept,
10681049
.get_port = inet_csk_get_port,

‎net/ipv4/devinet.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ static const struct nla_policy ifa_ipv4_policy[IFA_MAX+1] = {
107107

108108
static struct hlist_head inet_addr_lst[IN4_ADDR_HSIZE];
109109

110-
static u32 inet_addr_hash(struct net *net, __be32 addr)
110+
static u32 inet_addr_hash(const struct net *net, __be32 addr)
111111
{
112112
u32 val = (__force u32) addr ^ net_hash_mix(net);
113113

‎net/ipv4/inet_diag.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -742,14 +742,14 @@ static int inet_diag_dump_reqs(struct sk_buff *skb, struct sock *sk,
742742

743743
if (bc) {
744744
/* Note: entry.sport and entry.userlocks are already set */
745-
entry_fill_addrs(&entry, (struct sock *)req);
745+
entry_fill_addrs(&entry, req_to_sk(req));
746746
entry.dport = ntohs(ireq->ir_rmt_port);
747747

748748
if (!inet_diag_bc_run(bc, &entry))
749749
continue;
750750
}
751751

752-
err = inet_req_diag_fill((struct sock *)req, skb,
752+
err = inet_req_diag_fill(req_to_sk(req), skb,
753753
NETLINK_CB(cb->skb).portid,
754754
cb->nlh->nlmsg_seq,
755755
NLM_F_MULTI, cb->nlh);

‎net/ipv4/inet_hashtables.c

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@
2424
#include <net/secure_seq.h>
2525
#include <net/ip.h>
2626

27-
static unsigned int inet_ehashfn(struct net *net, const __be32 laddr,
28-
const __u16 lport, const __be32 faddr,
29-
const __be16 fport)
27+
static u32 inet_ehashfn(const struct net *net, const __be32 laddr,
28+
const __u16 lport, const __be32 faddr,
29+
const __be16 fport)
3030
{
3131
static u32 inet_ehash_secret __read_mostly;
3232

@@ -36,17 +36,21 @@ static unsigned int inet_ehashfn(struct net *net, const __be32 laddr,
3636
inet_ehash_secret + net_hash_mix(net));
3737
}
3838

39-
40-
static unsigned int inet_sk_ehashfn(const struct sock *sk)
39+
/* This function handles inet_sock, but also timewait and request sockets
40+
* for IPv4/IPv6.
41+
*/
42+
u32 sk_ehashfn(const struct sock *sk)
4143
{
42-
const struct inet_sock *inet = inet_sk(sk);
43-
const __be32 laddr = inet->inet_rcv_saddr;
44-
const __u16 lport = inet->inet_num;
45-
const __be32 faddr = inet->inet_daddr;
46-
const __be16 fport = inet->inet_dport;
47-
struct net *net = sock_net(sk);
48-
49-
return inet_ehashfn(net, laddr, lport, faddr, fport);
44+
#if IS_ENABLED(CONFIG_IPV6)
45+
if (sk->sk_family == AF_INET6 &&
46+
!ipv6_addr_v4mapped(&sk->sk_v6_daddr))
47+
return inet6_ehashfn(sock_net(sk),
48+
&sk->sk_v6_rcv_saddr, sk->sk_num,
49+
&sk->sk_v6_daddr, sk->sk_dport);
50+
#endif
51+
return inet_ehashfn(sock_net(sk),
52+
sk->sk_rcv_saddr, sk->sk_num,
53+
sk->sk_daddr, sk->sk_dport);
5054
}
5155

5256
/*
@@ -407,13 +411,13 @@ int __inet_hash_nolisten(struct sock *sk, struct inet_timewait_sock *tw)
407411
{
408412
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
409413
struct hlist_nulls_head *list;
410-
spinlock_t *lock;
411414
struct inet_ehash_bucket *head;
415+
spinlock_t *lock;
412416
int twrefcnt = 0;
413417

414418
WARN_ON(!sk_unhashed(sk));
415419

416-
sk->sk_hash = inet_sk_ehashfn(sk);
420+
sk->sk_hash = sk_ehashfn(sk);
417421
head = inet_ehash_bucket(hashinfo, sk->sk_hash);
418422
list = &head->chain;
419423
lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
@@ -430,15 +434,13 @@ int __inet_hash_nolisten(struct sock *sk, struct inet_timewait_sock *tw)
430434
}
431435
EXPORT_SYMBOL_GPL(__inet_hash_nolisten);
432436

433-
static void __inet_hash(struct sock *sk)
437+
int __inet_hash(struct sock *sk, struct inet_timewait_sock *tw)
434438
{
435439
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
436440
struct inet_listen_hashbucket *ilb;
437441

438-
if (sk->sk_state != TCP_LISTEN) {
439-
__inet_hash_nolisten(sk, NULL);
440-
return;
441-
}
442+
if (sk->sk_state != TCP_LISTEN)
443+
return __inet_hash_nolisten(sk, tw);
442444

443445
WARN_ON(!sk_unhashed(sk));
444446
ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
@@ -447,13 +449,15 @@ static void __inet_hash(struct sock *sk)
447449
__sk_nulls_add_node_rcu(sk, &ilb->head);
448450
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
449451
spin_unlock(&ilb->lock);
452+
return 0;
450453
}
454+
EXPORT_SYMBOL(__inet_hash);
451455

452456
void inet_hash(struct sock *sk)
453457
{
454458
if (sk->sk_state != TCP_CLOSE) {
455459
local_bh_disable();
456-
__inet_hash(sk);
460+
__inet_hash(sk, NULL);
457461
local_bh_enable();
458462
}
459463
}
@@ -484,8 +488,7 @@ EXPORT_SYMBOL_GPL(inet_unhash);
484488
int __inet_hash_connect(struct inet_timewait_death_row *death_row,
485489
struct sock *sk, u32 port_offset,
486490
int (*check_established)(struct inet_timewait_death_row *,
487-
struct sock *, __u16, struct inet_timewait_sock **),
488-
int (*hash)(struct sock *sk, struct inet_timewait_sock *twp))
491+
struct sock *, __u16, struct inet_timewait_sock **))
489492
{
490493
struct inet_hashinfo *hinfo = death_row->hashinfo;
491494
const unsigned short snum = inet_sk(sk)->inet_num;
@@ -555,7 +558,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
555558
inet_bind_hash(sk, tb, port);
556559
if (sk_unhashed(sk)) {
557560
inet_sk(sk)->inet_sport = htons(port);
558-
twrefcnt += hash(sk, tw);
561+
twrefcnt += __inet_hash_nolisten(sk, tw);
559562
}
560563
if (tw)
561564
twrefcnt += inet_twsk_bind_unhash(tw, hinfo);
@@ -577,7 +580,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
577580
tb = inet_csk(sk)->icsk_bind_hash;
578581
spin_lock_bh(&head->lock);
579582
if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
580-
hash(sk, NULL);
583+
__inet_hash_nolisten(sk, NULL);
581584
spin_unlock_bh(&head->lock);
582585
return 0;
583586
} else {
@@ -597,7 +600,7 @@ int inet_hash_connect(struct inet_timewait_death_row *death_row,
597600
struct sock *sk)
598601
{
599602
return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk),
600-
__inet_check_established, __inet_hash_nolisten);
603+
__inet_check_established);
601604
}
602605
EXPORT_SYMBOL_GPL(inet_hash_connect);
603606

‎net/ipv4/ping.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,11 @@ EXPORT_SYMBOL_GPL(pingv6_ops);
6464

6565
static u16 ping_port_rover;
6666

67-
static inline int ping_hashfn(struct net *net, unsigned int num, unsigned int mask)
67+
static inline u32 ping_hashfn(const struct net *net, u32 num, u32 mask)
6868
{
69-
int res = (num + net_hash_mix(net)) & mask;
69+
u32 res = (num + net_hash_mix(net)) & mask;
7070

71-
pr_debug("hash(%d) = %d\n", num, res);
71+
pr_debug("hash(%u) = %u\n", num, res);
7272
return res;
7373
}
7474
EXPORT_SYMBOL_GPL(ping_hash);

‎net/ipv4/syncookies.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,8 +337,8 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb)
337337
req->mss = mss;
338338
ireq->ir_num = ntohs(th->dest);
339339
ireq->ir_rmt_port = th->source;
340-
ireq->ir_loc_addr = ip_hdr(skb)->daddr;
341-
ireq->ir_rmt_addr = ip_hdr(skb)->saddr;
340+
sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
341+
sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
342342
ireq->ir_mark = inet_request_mark(sk, skb);
343343
ireq->snd_wscale = tcp_opt.snd_wscale;
344344
ireq->sack_ok = tcp_opt.sack_ok;

0 commit comments

Comments
 (0)