2005-08-10 11:00:51 +08:00
|
|
|
/*
|
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
|
*
|
|
|
|
|
* Generic INET transport hashtables
|
|
|
|
|
*
|
|
|
|
|
* Authors: Lotsa people, from code originally in tcp
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
|
*/
|
|
|
|
|
|
2005-08-10 11:07:13 +08:00
|
|
|
#include <linux/module.h>
|
2005-12-14 15:25:31 +08:00
|
|
|
#include <linux/random.h>
|
2005-08-10 11:08:09 +08:00
|
|
|
#include <linux/sched.h>
|
2005-08-10 11:00:51 +08:00
|
|
|
#include <linux/slab.h>
|
2005-08-10 11:08:09 +08:00
|
|
|
#include <linux/wait.h>
|
2015-05-26 22:55:34 +08:00
|
|
|
#include <linux/vmalloc.h>
|
2005-08-10 11:00:51 +08:00
|
|
|
|
2005-08-10 11:10:42 +08:00
|
|
|
#include <net/inet_connection_sock.h>
|
2005-08-10 11:00:51 +08:00
|
|
|
#include <net/inet_hashtables.h>
|
2011-08-04 11:50:44 +08:00
|
|
|
#include <net/secure_seq.h>
|
2005-12-14 15:25:31 +08:00
|
|
|
#include <net/ip.h>
|
2005-08-10 11:00:51 +08:00
|
|
|
|
2015-03-19 05:05:33 +08:00
|
|
|
static u32 inet_ehashfn(const struct net *net, const __be32 laddr,
|
|
|
|
|
const __u16 lport, const __be32 faddr,
|
|
|
|
|
const __be16 fport)
|
2013-10-20 03:48:51 +08:00
|
|
|
{
|
2013-10-20 03:48:57 +08:00
|
|
|
static u32 inet_ehash_secret __read_mostly;
|
|
|
|
|
|
|
|
|
|
net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));
|
|
|
|
|
|
2013-10-20 03:48:51 +08:00
|
|
|
return __inet_ehashfn(laddr, lport, faddr, fport,
|
|
|
|
|
inet_ehash_secret + net_hash_mix(net));
|
|
|
|
|
}
|
|
|
|
|
|
2015-03-19 05:05:35 +08:00
|
|
|
/* This function handles inet_sock, but also timewait and request sockets
|
|
|
|
|
* for IPv4/IPv6.
|
|
|
|
|
*/
|
2015-03-19 05:05:34 +08:00
|
|
|
u32 sk_ehashfn(const struct sock *sk)
|
2013-10-20 03:48:51 +08:00
|
|
|
{
|
2015-03-19 05:05:35 +08:00
|
|
|
#if IS_ENABLED(CONFIG_IPV6)
|
|
|
|
|
if (sk->sk_family == AF_INET6 &&
|
|
|
|
|
!ipv6_addr_v4mapped(&sk->sk_v6_daddr))
|
|
|
|
|
return inet6_ehashfn(sock_net(sk),
|
|
|
|
|
&sk->sk_v6_rcv_saddr, sk->sk_num,
|
|
|
|
|
&sk->sk_v6_daddr, sk->sk_dport);
|
|
|
|
|
#endif
|
2015-03-19 05:05:34 +08:00
|
|
|
return inet_ehashfn(sock_net(sk),
|
|
|
|
|
sk->sk_rcv_saddr, sk->sk_num,
|
|
|
|
|
sk->sk_daddr, sk->sk_dport);
|
2013-10-20 03:48:51 +08:00
|
|
|
}
|
|
|
|
|
|
2005-08-10 11:00:51 +08:00
|
|
|
/*
|
|
|
|
|
* Allocate and initialize a new local port bind bucket.
|
|
|
|
|
* The bindhash mutex for snum's hash chain must be held here.
|
|
|
|
|
*/
|
2006-12-07 12:33:20 +08:00
|
|
|
struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
|
2008-01-31 21:05:50 +08:00
|
|
|
struct net *net,
|
2005-08-10 11:00:51 +08:00
|
|
|
struct inet_bind_hashbucket *head,
|
|
|
|
|
const unsigned short snum)
|
|
|
|
|
{
|
2006-12-07 12:33:16 +08:00
|
|
|
struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
|
2005-08-10 11:00:51 +08:00
|
|
|
|
2015-04-03 16:17:27 +08:00
|
|
|
if (tb) {
|
2015-03-12 12:04:08 +08:00
|
|
|
write_pnet(&tb->ib_net, net);
|
2005-08-10 11:00:51 +08:00
|
|
|
tb->port = snum;
|
|
|
|
|
tb->fastreuse = 0;
|
2013-01-22 17:50:24 +08:00
|
|
|
tb->fastreuseport = 0;
|
inet: Allowing more than 64k connections and heavily optimize bind(0) time.
With simple extension to the binding mechanism, which allows to bind more
than 64k sockets (or smaller amount, depending on sysctl parameters),
we have to traverse the whole bind hash table to find out empty bucket.
And while it is not a problem for example for 32k connections, bind()
completion time grows exponentially (since after each successful binding
we have to traverse one bucket more to find empty one) even if we start
each time from random offset inside the hash table.
So, when hash table is full, and we want to add another socket, we have
to traverse the whole table no matter what, so effectivelly this will be
the worst case performance and it will be constant.
Attached picture shows bind() time depending on number of already bound
sockets.
Green area corresponds to the usual binding to zero port process, which
turns on kernel port selection as described above. Red area is the bind
process, when number of reuse-bound sockets is not limited by 64k (or
sysctl parameters). The same exponential growth (hidden by the green
area) before number of ports reaches sysctl limit.
At this time bind hash table has exactly one reuse-enbaled socket in a
bucket, but it is possible that they have different addresses. Actually
kernel selects the first port to try randomly, so at the beginning bind
will take roughly constant time, but with time number of port to check
after random start will increase. And that will have exponential growth,
but because of above random selection, not every next port selection
will necessary take longer time than previous. So we have to consider
the area below in the graph (if you could zoom it, you could find, that
there are many different times placed there), so area can hide another.
Blue area corresponds to the port selection optimization.
This is rather simple design approach: hashtable now maintains (unprecise
and racely updated) number of currently bound sockets, and when number
of such sockets becomes greater than predefined value (I use maximum
port range defined by sysctls), we stop traversing the whole bind hash
table and just stop at first matching bucket after random start. Above
limit roughly corresponds to the case, when bind hash table is full and
we turned on mechanism of allowing to bind more reuse-enabled sockets,
so it does not change behaviour of other sockets.
Signed-off-by: Evgeniy Polyakov <zbr@ioremap.net>
Tested-by: Denys Fedoryschenko <denys@visp.net.lb>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-01-20 08:46:02 +08:00
|
|
|
tb->num_owners = 0;
|
2005-08-10 11:00:51 +08:00
|
|
|
INIT_HLIST_HEAD(&tb->owners);
|
|
|
|
|
hlist_add_head(&tb->node, &head->chain);
|
|
|
|
|
}
|
|
|
|
|
return tb;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Caller must hold hashbucket lock for this tb with local BH disabled
|
|
|
|
|
*/
|
2006-12-07 12:33:20 +08:00
|
|
|
void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
|
2005-08-10 11:00:51 +08:00
|
|
|
{
|
|
|
|
|
if (hlist_empty(&tb->owners)) {
|
|
|
|
|
__hlist_del(&tb->node);
|
|
|
|
|
kmem_cache_free(cachep, tb);
|
|
|
|
|
}
|
|
|
|
|
}
|
2005-08-10 11:07:13 +08:00
|
|
|
|
|
|
|
|
void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
|
|
|
|
|
const unsigned short snum)
|
|
|
|
|
{
|
2009-10-15 14:30:45 +08:00
|
|
|
inet_sk(sk)->inet_num = snum;
|
2005-08-10 11:07:13 +08:00
|
|
|
sk_add_bind_node(sk, &tb->owners);
|
inet: Allowing more than 64k connections and heavily optimize bind(0) time.
With simple extension to the binding mechanism, which allows to bind more
than 64k sockets (or smaller amount, depending on sysctl parameters),
we have to traverse the whole bind hash table to find out empty bucket.
And while it is not a problem for example for 32k connections, bind()
completion time grows exponentially (since after each successful binding
we have to traverse one bucket more to find empty one) even if we start
each time from random offset inside the hash table.
So, when hash table is full, and we want to add another socket, we have
to traverse the whole table no matter what, so effectivelly this will be
the worst case performance and it will be constant.
Attached picture shows bind() time depending on number of already bound
sockets.
Green area corresponds to the usual binding to zero port process, which
turns on kernel port selection as described above. Red area is the bind
process, when number of reuse-bound sockets is not limited by 64k (or
sysctl parameters). The same exponential growth (hidden by the green
area) before number of ports reaches sysctl limit.
At this time bind hash table has exactly one reuse-enbaled socket in a
bucket, but it is possible that they have different addresses. Actually
kernel selects the first port to try randomly, so at the beginning bind
will take roughly constant time, but with time number of port to check
after random start will increase. And that will have exponential growth,
but because of above random selection, not every next port selection
will necessary take longer time than previous. So we have to consider
the area below in the graph (if you could zoom it, you could find, that
there are many different times placed there), so area can hide another.
Blue area corresponds to the port selection optimization.
This is rather simple design approach: hashtable now maintains (unprecise
and racely updated) number of currently bound sockets, and when number
of such sockets becomes greater than predefined value (I use maximum
port range defined by sysctls), we stop traversing the whole bind hash
table and just stop at first matching bucket after random start. Above
limit roughly corresponds to the case, when bind hash table is full and
we turned on mechanism of allowing to bind more reuse-enabled sockets,
so it does not change behaviour of other sockets.
Signed-off-by: Evgeniy Polyakov <zbr@ioremap.net>
Tested-by: Denys Fedoryschenko <denys@visp.net.lb>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-01-20 08:46:02 +08:00
|
|
|
tb->num_owners++;
|
2005-08-10 11:10:42 +08:00
|
|
|
inet_csk(sk)->icsk_bind_hash = tb;
|
2005-08-10 11:07:13 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Get rid of any references to a local port held by the given sock.
|
|
|
|
|
*/
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
static void __inet_put_port(struct sock *sk)
|
2005-08-10 11:07:13 +08:00
|
|
|
{
|
2008-03-23 07:50:58 +08:00
|
|
|
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
|
2009-10-15 14:30:45 +08:00
|
|
|
const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
|
2008-06-17 08:12:49 +08:00
|
|
|
hashinfo->bhash_size);
|
2005-08-10 11:07:13 +08:00
|
|
|
struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
|
|
|
|
|
struct inet_bind_bucket *tb;
|
|
|
|
|
|
|
|
|
|
spin_lock(&head->lock);
|
2005-08-10 11:10:42 +08:00
|
|
|
tb = inet_csk(sk)->icsk_bind_hash;
|
2005-08-10 11:07:13 +08:00
|
|
|
__sk_del_bind_node(sk);
|
inet: Allowing more than 64k connections and heavily optimize bind(0) time.
With simple extension to the binding mechanism, which allows to bind more
than 64k sockets (or smaller amount, depending on sysctl parameters),
we have to traverse the whole bind hash table to find out empty bucket.
And while it is not a problem for example for 32k connections, bind()
completion time grows exponentially (since after each successful binding
we have to traverse one bucket more to find empty one) even if we start
each time from random offset inside the hash table.
So, when hash table is full, and we want to add another socket, we have
to traverse the whole table no matter what, so effectivelly this will be
the worst case performance and it will be constant.
Attached picture shows bind() time depending on number of already bound
sockets.
Green area corresponds to the usual binding to zero port process, which
turns on kernel port selection as described above. Red area is the bind
process, when number of reuse-bound sockets is not limited by 64k (or
sysctl parameters). The same exponential growth (hidden by the green
area) before number of ports reaches sysctl limit.
At this time bind hash table has exactly one reuse-enbaled socket in a
bucket, but it is possible that they have different addresses. Actually
kernel selects the first port to try randomly, so at the beginning bind
will take roughly constant time, but with time number of port to check
after random start will increase. And that will have exponential growth,
but because of above random selection, not every next port selection
will necessary take longer time than previous. So we have to consider
the area below in the graph (if you could zoom it, you could find, that
there are many different times placed there), so area can hide another.
Blue area corresponds to the port selection optimization.
This is rather simple design approach: hashtable now maintains (unprecise
and racely updated) number of currently bound sockets, and when number
of such sockets becomes greater than predefined value (I use maximum
port range defined by sysctls), we stop traversing the whole bind hash
table and just stop at first matching bucket after random start. Above
limit roughly corresponds to the case, when bind hash table is full and
we turned on mechanism of allowing to bind more reuse-enabled sockets,
so it does not change behaviour of other sockets.
Signed-off-by: Evgeniy Polyakov <zbr@ioremap.net>
Tested-by: Denys Fedoryschenko <denys@visp.net.lb>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-01-20 08:46:02 +08:00
|
|
|
tb->num_owners--;
|
2005-08-10 11:10:42 +08:00
|
|
|
inet_csk(sk)->icsk_bind_hash = NULL;
|
2009-10-15 14:30:45 +08:00
|
|
|
inet_sk(sk)->inet_num = 0;
|
2005-08-10 11:07:13 +08:00
|
|
|
inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
|
|
|
|
|
spin_unlock(&head->lock);
|
|
|
|
|
}
|
|
|
|
|
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
void inet_put_port(struct sock *sk)
|
2005-08-10 11:07:13 +08:00
|
|
|
{
|
|
|
|
|
local_bh_disable();
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
__inet_put_port(sk);
|
2005-08-10 11:07:13 +08:00
|
|
|
local_bh_enable();
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(inet_put_port);
|
2005-08-10 11:08:09 +08:00
|
|
|
|
2015-09-29 22:42:44 +08:00
|
|
|
int __inet_inherit_port(const struct sock *sk, struct sock *child)
|
2008-04-18 14:18:15 +08:00
|
|
|
{
|
|
|
|
|
struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
|
2010-10-21 19:06:43 +08:00
|
|
|
unsigned short port = inet_sk(child)->inet_num;
|
|
|
|
|
const int bhash = inet_bhashfn(sock_net(sk), port,
|
2008-06-17 08:12:49 +08:00
|
|
|
table->bhash_size);
|
2008-04-18 14:18:15 +08:00
|
|
|
struct inet_bind_hashbucket *head = &table->bhash[bhash];
|
|
|
|
|
struct inet_bind_bucket *tb;
|
|
|
|
|
|
|
|
|
|
spin_lock(&head->lock);
|
|
|
|
|
tb = inet_csk(sk)->icsk_bind_hash;
|
2015-10-14 20:58:38 +08:00
|
|
|
if (unlikely(!tb)) {
|
|
|
|
|
spin_unlock(&head->lock);
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
}
|
2010-10-21 19:06:43 +08:00
|
|
|
if (tb->port != port) {
|
|
|
|
|
/* NOTE: using tproxy and redirecting skbs to a proxy
|
|
|
|
|
* on a different listener port breaks the assumption
|
|
|
|
|
* that the listener socket's icsk_bind_hash is the same
|
|
|
|
|
* as that of the child socket. We have to look up or
|
|
|
|
|
* create a new bind bucket for the child here. */
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 09:06:00 +08:00
|
|
|
inet_bind_bucket_for_each(tb, &head->chain) {
|
2010-10-21 19:06:43 +08:00
|
|
|
if (net_eq(ib_net(tb), sock_net(sk)) &&
|
|
|
|
|
tb->port == port)
|
|
|
|
|
break;
|
|
|
|
|
}
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 09:06:00 +08:00
|
|
|
if (!tb) {
|
2010-10-21 19:06:43 +08:00
|
|
|
tb = inet_bind_bucket_create(table->bind_bucket_cachep,
|
|
|
|
|
sock_net(sk), head, port);
|
|
|
|
|
if (!tb) {
|
|
|
|
|
spin_unlock(&head->lock);
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2010-11-26 22:26:27 +08:00
|
|
|
inet_bind_hash(child, tb, port);
|
2008-04-18 14:18:15 +08:00
|
|
|
spin_unlock(&head->lock);
|
2010-10-21 19:06:43 +08:00
|
|
|
|
|
|
|
|
return 0;
|
2008-04-18 14:18:15 +08:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(__inet_inherit_port);
|
|
|
|
|
|
2008-11-24 09:22:55 +08:00
|
|
|
static inline int compute_score(struct sock *sk, struct net *net,
|
|
|
|
|
const unsigned short hnum, const __be32 daddr,
|
|
|
|
|
const int dif)
|
|
|
|
|
{
|
|
|
|
|
int score = -1;
|
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
|
|
2009-10-15 14:30:45 +08:00
|
|
|
if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&
|
2008-11-24 09:22:55 +08:00
|
|
|
!ipv6_only_sock(sk)) {
|
2009-10-15 14:30:45 +08:00
|
|
|
__be32 rcv_saddr = inet->inet_rcv_saddr;
|
2013-01-22 17:50:24 +08:00
|
|
|
score = sk->sk_family == PF_INET ? 2 : 1;
|
2008-11-24 09:22:55 +08:00
|
|
|
if (rcv_saddr) {
|
|
|
|
|
if (rcv_saddr != daddr)
|
|
|
|
|
return -1;
|
2013-01-22 17:50:24 +08:00
|
|
|
score += 4;
|
2008-11-24 09:22:55 +08:00
|
|
|
}
|
|
|
|
|
if (sk->sk_bound_dev_if) {
|
|
|
|
|
if (sk->sk_bound_dev_if != dif)
|
|
|
|
|
return -1;
|
2013-01-22 17:50:24 +08:00
|
|
|
score += 4;
|
2008-11-24 09:22:55 +08:00
|
|
|
}
|
2015-10-09 10:33:21 +08:00
|
|
|
if (sk->sk_incoming_cpu == raw_smp_processor_id())
|
|
|
|
|
score++;
|
2008-11-24 09:22:55 +08:00
|
|
|
}
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
2005-08-10 11:09:06 +08:00
|
|
|
/*
|
|
|
|
|
* Don't inline this cruft. Here are some nice properties to exploit here. The
|
|
|
|
|
* BSD API does not allow a listening sock to specify the remote port nor the
|
|
|
|
|
* remote address for the connection. So always assume those are both
|
|
|
|
|
* wildcarded during the search since they can never be otherwise.
|
|
|
|
|
*/
|
2005-08-10 11:09:46 +08:00
|
|
|
|
2008-11-24 09:22:55 +08:00
|
|
|
|
2008-01-31 21:06:40 +08:00
|
|
|
struct sock *__inet_lookup_listener(struct net *net,
|
|
|
|
|
struct inet_hashinfo *hashinfo,
|
2013-01-22 17:50:24 +08:00
|
|
|
const __be32 saddr, __be16 sport,
|
2006-09-28 09:43:33 +08:00
|
|
|
const __be32 daddr, const unsigned short hnum,
|
2006-08-10 06:47:12 +08:00
|
|
|
const int dif)
|
2006-08-08 17:18:10 +08:00
|
|
|
{
|
2008-11-24 09:22:55 +08:00
|
|
|
struct sock *sk, *result;
|
|
|
|
|
struct hlist_nulls_node *node;
|
|
|
|
|
unsigned int hash = inet_lhashfn(net, hnum);
|
|
|
|
|
struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
|
2013-01-22 17:50:24 +08:00
|
|
|
int score, hiscore, matches = 0, reuseport = 0;
|
|
|
|
|
u32 phash = 0;
|
2006-08-08 17:18:10 +08:00
|
|
|
|
2008-11-24 09:22:55 +08:00
|
|
|
rcu_read_lock();
|
|
|
|
|
begin:
|
|
|
|
|
result = NULL;
|
2013-01-22 17:50:24 +08:00
|
|
|
hiscore = 0;
|
2008-11-24 09:22:55 +08:00
|
|
|
sk_nulls_for_each_rcu(sk, node, &ilb->head) {
|
|
|
|
|
score = compute_score(sk, net, hnum, daddr, dif);
|
|
|
|
|
if (score > hiscore) {
|
|
|
|
|
result = sk;
|
|
|
|
|
hiscore = score;
|
2013-01-22 17:50:24 +08:00
|
|
|
reuseport = sk->sk_reuseport;
|
|
|
|
|
if (reuseport) {
|
|
|
|
|
phash = inet_ehashfn(net, daddr, hnum,
|
|
|
|
|
saddr, sport);
|
|
|
|
|
matches = 1;
|
|
|
|
|
}
|
|
|
|
|
} else if (score == hiscore && reuseport) {
|
|
|
|
|
matches++;
|
2014-08-24 02:58:54 +08:00
|
|
|
if (reciprocal_scale(phash, matches) == 0)
|
2013-01-22 17:50:24 +08:00
|
|
|
result = sk;
|
|
|
|
|
phash = next_pseudo_random32(phash);
|
2008-11-24 09:22:55 +08:00
|
|
|
}
|
2006-08-08 17:18:10 +08:00
|
|
|
}
|
2008-11-24 09:22:55 +08:00
|
|
|
/*
|
|
|
|
|
* if the nulls value we got at the end of this lookup is
|
|
|
|
|
* not the expected one, we must restart lookup.
|
|
|
|
|
* We probably met an item that was moved to another chain.
|
|
|
|
|
*/
|
|
|
|
|
if (get_nulls_value(node) != hash + LISTENING_NULLS_BASE)
|
|
|
|
|
goto begin;
|
|
|
|
|
if (result) {
|
|
|
|
|
if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
|
|
|
|
|
result = NULL;
|
|
|
|
|
else if (unlikely(compute_score(result, net, hnum, daddr,
|
|
|
|
|
dif) < hiscore)) {
|
|
|
|
|
sock_put(result);
|
|
|
|
|
goto begin;
|
|
|
|
|
}
|
2006-08-08 17:18:10 +08:00
|
|
|
}
|
2008-11-24 09:22:55 +08:00
|
|
|
rcu_read_unlock();
|
|
|
|
|
return result;
|
2006-08-08 17:18:10 +08:00
|
|
|
}
|
2006-08-10 06:47:12 +08:00
|
|
|
EXPORT_SYMBOL_GPL(__inet_lookup_listener);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
/* All sockets share common refcount, but have different destructors */
|
|
|
|
|
void sock_gen_put(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
if (!atomic_dec_and_test(&sk->sk_refcnt))
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
if (sk->sk_state == TCP_TIME_WAIT)
|
|
|
|
|
inet_twsk_free(inet_twsk(sk));
|
2015-03-13 07:44:08 +08:00
|
|
|
else if (sk->sk_state == TCP_NEW_SYN_RECV)
|
|
|
|
|
reqsk_free(inet_reqsk(sk));
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
else
|
|
|
|
|
sk_free(sk);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(sock_gen_put);
|
|
|
|
|
|
2015-03-16 12:12:15 +08:00
|
|
|
void sock_edemux(struct sk_buff *skb)
|
|
|
|
|
{
|
|
|
|
|
sock_gen_put(skb->sk);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(sock_edemux);
|
|
|
|
|
|
2012-04-15 09:34:41 +08:00
|
|
|
struct sock *__inet_lookup_established(struct net *net,
|
2008-01-31 21:06:40 +08:00
|
|
|
struct inet_hashinfo *hashinfo,
|
2007-12-21 07:32:17 +08:00
|
|
|
const __be32 saddr, const __be16 sport,
|
|
|
|
|
const __be32 daddr, const u16 hnum,
|
|
|
|
|
const int dif)
|
|
|
|
|
{
|
2014-05-14 11:30:07 +08:00
|
|
|
INET_ADDR_COOKIE(acookie, saddr, daddr);
|
2007-12-21 07:32:17 +08:00
|
|
|
const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
|
|
|
|
|
struct sock *sk;
|
2008-11-17 11:40:17 +08:00
|
|
|
const struct hlist_nulls_node *node;
|
2007-12-21 07:32:17 +08:00
|
|
|
/* Optimize here for direct hit, only listening connections can
|
|
|
|
|
* have wildcards anyways.
|
|
|
|
|
*/
|
2008-06-17 08:13:27 +08:00
|
|
|
unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
|
2009-10-09 08:16:19 +08:00
|
|
|
unsigned int slot = hash & hashinfo->ehash_mask;
|
2008-11-17 11:40:17 +08:00
|
|
|
struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
|
2007-12-21 07:32:17 +08:00
|
|
|
|
2008-11-17 11:40:17 +08:00
|
|
|
rcu_read_lock();
|
|
|
|
|
begin:
|
|
|
|
|
sk_nulls_for_each_rcu(sk, node, &head->chain) {
|
2012-11-30 17:49:27 +08:00
|
|
|
if (sk->sk_hash != hash)
|
|
|
|
|
continue;
|
|
|
|
|
if (likely(INET_MATCH(sk, net, acookie,
|
|
|
|
|
saddr, daddr, ports, dif))) {
|
2008-11-17 11:40:17 +08:00
|
|
|
if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
goto out;
|
2012-11-30 17:49:27 +08:00
|
|
|
if (unlikely(!INET_MATCH(sk, net, acookie,
|
|
|
|
|
saddr, daddr, ports, dif))) {
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
sock_gen_put(sk);
|
2008-11-17 11:40:17 +08:00
|
|
|
goto begin;
|
|
|
|
|
}
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
goto found;
|
2008-11-17 11:40:17 +08:00
|
|
|
}
|
2007-12-21 07:32:17 +08:00
|
|
|
}
|
2008-11-17 11:40:17 +08:00
|
|
|
/*
|
|
|
|
|
* if the nulls value we got at the end of this lookup is
|
|
|
|
|
* not the expected one, we must restart lookup.
|
|
|
|
|
* We probably met an item that was moved to another chain.
|
|
|
|
|
*/
|
|
|
|
|
if (get_nulls_value(node) != slot)
|
|
|
|
|
goto begin;
|
2007-12-21 07:32:17 +08:00
|
|
|
out:
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
sk = NULL;
|
|
|
|
|
found:
|
2008-11-17 11:40:17 +08:00
|
|
|
rcu_read_unlock();
|
2007-12-21 07:32:17 +08:00
|
|
|
return sk;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(__inet_lookup_established);
|
|
|
|
|
|
2005-12-14 15:25:31 +08:00
|
|
|
/* called with local bh disabled */
|
|
|
|
|
static int __inet_check_established(struct inet_timewait_death_row *death_row,
|
|
|
|
|
struct sock *sk, __u16 lport,
|
|
|
|
|
struct inet_timewait_sock **twp)
|
|
|
|
|
{
|
|
|
|
|
struct inet_hashinfo *hinfo = death_row->hashinfo;
|
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2009-10-15 14:30:45 +08:00
|
|
|
__be32 daddr = inet->inet_rcv_saddr;
|
|
|
|
|
__be32 saddr = inet->inet_daddr;
|
2005-12-14 15:25:31 +08:00
|
|
|
int dif = sk->sk_bound_dev_if;
|
2014-05-14 11:30:07 +08:00
|
|
|
INET_ADDR_COOKIE(acookie, saddr, daddr);
|
2009-10-15 14:30:45 +08:00
|
|
|
const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
|
2008-06-17 08:13:27 +08:00
|
|
|
struct net *net = sock_net(sk);
|
2009-10-15 14:30:45 +08:00
|
|
|
unsigned int hash = inet_ehashfn(net, daddr, lport,
|
|
|
|
|
saddr, inet->inet_dport);
|
2005-12-14 15:25:31 +08:00
|
|
|
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
|
2008-11-21 12:39:09 +08:00
|
|
|
spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
|
2005-12-14 15:25:31 +08:00
|
|
|
struct sock *sk2;
|
2008-11-17 11:40:17 +08:00
|
|
|
const struct hlist_nulls_node *node;
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
struct inet_timewait_sock *tw = NULL;
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_lock(lock);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2008-11-17 11:40:17 +08:00
|
|
|
sk_nulls_for_each(sk2, node, &head->chain) {
|
2012-11-30 17:49:27 +08:00
|
|
|
if (sk2->sk_hash != hash)
|
|
|
|
|
continue;
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
|
2012-11-30 17:49:27 +08:00
|
|
|
if (likely(INET_MATCH(sk2, net, acookie,
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
saddr, daddr, ports, dif))) {
|
|
|
|
|
if (sk2->sk_state == TCP_TIME_WAIT) {
|
|
|
|
|
tw = inet_twsk(sk2);
|
|
|
|
|
if (twsk_unique(sk, sk2, twp))
|
|
|
|
|
break;
|
|
|
|
|
}
|
2005-12-14 15:25:31 +08:00
|
|
|
goto not_unique;
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
}
|
2005-12-14 15:25:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Must record num and sport now. Otherwise we will see
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
* in hash table socket with a funny identity.
|
|
|
|
|
*/
|
2009-10-15 14:30:45 +08:00
|
|
|
inet->inet_num = lport;
|
|
|
|
|
inet->inet_sport = htons(lport);
|
2005-12-14 15:25:31 +08:00
|
|
|
sk->sk_hash = hash;
|
2008-07-26 12:43:18 +08:00
|
|
|
WARN_ON(!sk_unhashed(sk));
|
2008-11-17 11:40:17 +08:00
|
|
|
__sk_nulls_add_node_rcu(sk, &head->chain);
|
2009-12-03 06:31:19 +08:00
|
|
|
if (tw) {
|
2015-07-09 05:28:29 +08:00
|
|
|
sk_nulls_del_node_init_rcu((struct sock *)tw);
|
2009-12-03 06:31:19 +08:00
|
|
|
NET_INC_STATS_BH(net, LINUX_MIB_TIMEWAITRECYCLED);
|
|
|
|
|
}
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_unlock(lock);
|
2008-04-01 10:41:46 +08:00
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
|
|
|
|
if (twp) {
|
|
|
|
|
*twp = tw;
|
|
|
|
|
} else if (tw) {
|
|
|
|
|
/* Silly. Should hash-dance instead... */
|
2015-07-09 05:28:30 +08:00
|
|
|
inet_twsk_deschedule_put(tw);
|
2005-12-14 15:25:31 +08:00
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
not_unique:
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_unlock(lock);
|
2005-12-14 15:25:31 +08:00
|
|
|
return -EADDRNOTAVAIL;
|
|
|
|
|
}
|
|
|
|
|
|
2015-05-28 01:46:02 +08:00
|
|
|
static u32 inet_sk_port_offset(const struct sock *sk)
|
2005-12-14 15:25:31 +08:00
|
|
|
{
|
|
|
|
|
const struct inet_sock *inet = inet_sk(sk);
|
2015-05-28 01:46:02 +08:00
|
|
|
|
2009-10-15 14:30:45 +08:00
|
|
|
return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr,
|
|
|
|
|
inet->inet_daddr,
|
|
|
|
|
inet->inet_dport);
|
2005-12-14 15:25:31 +08:00
|
|
|
}
|
|
|
|
|
|
2015-10-03 02:43:32 +08:00
|
|
|
/* insert a socket into ehash, and eventually remove another one
|
|
|
|
|
* (The another one can be a SYN_RECV or TIMEWAIT
|
|
|
|
|
*/
|
2015-10-22 23:20:46 +08:00
|
|
|
bool inet_ehash_insert(struct sock *sk, struct sock *osk)
|
2007-12-21 07:31:33 +08:00
|
|
|
{
|
2008-03-23 07:50:58 +08:00
|
|
|
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
|
2008-11-17 11:40:17 +08:00
|
|
|
struct hlist_nulls_head *list;
|
2007-12-21 07:31:33 +08:00
|
|
|
struct inet_ehash_bucket *head;
|
2015-03-19 05:05:34 +08:00
|
|
|
spinlock_t *lock;
|
2015-10-22 23:20:46 +08:00
|
|
|
bool ret = true;
|
2007-12-21 07:31:33 +08:00
|
|
|
|
2015-10-03 02:43:32 +08:00
|
|
|
WARN_ON_ONCE(!sk_unhashed(sk));
|
2007-12-21 07:31:33 +08:00
|
|
|
|
2015-03-19 05:05:34 +08:00
|
|
|
sk->sk_hash = sk_ehashfn(sk);
|
2007-12-21 07:31:33 +08:00
|
|
|
head = inet_ehash_bucket(hashinfo, sk->sk_hash);
|
|
|
|
|
list = &head->chain;
|
|
|
|
|
lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
|
|
|
|
|
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_lock(lock);
|
2015-07-09 05:28:29 +08:00
|
|
|
if (osk) {
|
2015-10-22 23:20:46 +08:00
|
|
|
WARN_ON_ONCE(sk->sk_hash != osk->sk_hash);
|
|
|
|
|
ret = sk_nulls_del_node_init_rcu(osk);
|
2009-12-04 11:46:54 +08:00
|
|
|
}
|
2015-10-22 23:20:46 +08:00
|
|
|
if (ret)
|
|
|
|
|
__sk_nulls_add_node_rcu(sk, list);
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_unlock(lock);
|
2015-10-03 02:43:32 +08:00
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2015-10-22 23:20:46 +08:00
|
|
|
bool inet_ehash_nolisten(struct sock *sk, struct sock *osk)
|
2015-10-03 02:43:32 +08:00
|
|
|
{
|
2015-10-22 23:20:46 +08:00
|
|
|
bool ok = inet_ehash_insert(sk, osk);
|
|
|
|
|
|
|
|
|
|
if (ok) {
|
|
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
|
|
|
|
|
} else {
|
|
|
|
|
percpu_counter_inc(sk->sk_prot->orphan_count);
|
|
|
|
|
sk->sk_state = TCP_CLOSE;
|
|
|
|
|
sock_set_flag(sk, SOCK_DEAD);
|
|
|
|
|
inet_csk_destroy_sock(sk);
|
|
|
|
|
}
|
|
|
|
|
return ok;
|
2007-12-21 07:31:33 +08:00
|
|
|
}
|
2015-10-22 23:20:46 +08:00
|
|
|
EXPORT_SYMBOL_GPL(inet_ehash_nolisten);
|
2007-12-21 07:31:33 +08:00
|
|
|
|
2015-07-09 05:28:29 +08:00
|
|
|
void __inet_hash(struct sock *sk, struct sock *osk)
|
2007-12-21 07:31:33 +08:00
|
|
|
{
|
2008-03-23 07:50:58 +08:00
|
|
|
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
|
2008-11-20 16:40:07 +08:00
|
|
|
struct inet_listen_hashbucket *ilb;
|
2007-12-21 07:31:33 +08:00
|
|
|
|
2015-10-22 23:20:46 +08:00
|
|
|
if (sk->sk_state != TCP_LISTEN) {
|
|
|
|
|
inet_ehash_nolisten(sk, osk);
|
|
|
|
|
return;
|
|
|
|
|
}
|
2008-07-26 12:43:18 +08:00
|
|
|
WARN_ON(!sk_unhashed(sk));
|
2008-11-20 16:40:07 +08:00
|
|
|
ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
|
2007-12-21 07:31:33 +08:00
|
|
|
|
2008-11-20 16:40:07 +08:00
|
|
|
spin_lock(&ilb->lock);
|
2008-11-24 09:22:55 +08:00
|
|
|
__sk_nulls_add_node_rcu(sk, &ilb->head);
|
2008-04-01 10:41:46 +08:00
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
|
2008-11-20 16:40:07 +08:00
|
|
|
spin_unlock(&ilb->lock);
|
2007-12-21 07:31:33 +08:00
|
|
|
}
|
2015-03-19 05:05:36 +08:00
|
|
|
EXPORT_SYMBOL(__inet_hash);
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
|
|
|
|
|
void inet_hash(struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
if (sk->sk_state != TCP_CLOSE) {
|
|
|
|
|
local_bh_disable();
|
2015-03-19 05:05:36 +08:00
|
|
|
__inet_hash(sk, NULL);
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
local_bh_enable();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(inet_hash);
|
|
|
|
|
|
|
|
|
|
void inet_unhash(struct sock *sk)
|
|
|
|
|
{
|
2008-03-23 07:50:58 +08:00
|
|
|
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
|
2008-11-24 09:22:55 +08:00
|
|
|
spinlock_t *lock;
|
|
|
|
|
int done;
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
|
|
|
|
|
if (sk_unhashed(sk))
|
2008-11-20 16:40:07 +08:00
|
|
|
return;
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
|
2008-11-24 09:22:55 +08:00
|
|
|
if (sk->sk_state == TCP_LISTEN)
|
|
|
|
|
lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;
|
|
|
|
|
else
|
|
|
|
|
lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
|
2008-11-20 16:40:07 +08:00
|
|
|
|
2008-11-24 09:22:55 +08:00
|
|
|
spin_lock_bh(lock);
|
2013-07-11 14:55:51 +08:00
|
|
|
done = __sk_nulls_del_node_init_rcu(sk);
|
2008-11-24 09:22:55 +08:00
|
|
|
if (done)
|
|
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
|
2008-11-24 16:09:29 +08:00
|
|
|
spin_unlock_bh(lock);
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(inet_unhash);
|
2007-12-21 07:31:33 +08:00
|
|
|
|
2008-01-31 21:04:45 +08:00
|
|
|
int __inet_hash_connect(struct inet_timewait_death_row *death_row,
|
2008-02-05 19:14:44 +08:00
|
|
|
struct sock *sk, u32 port_offset,
|
2008-01-31 21:04:45 +08:00
|
|
|
int (*check_established)(struct inet_timewait_death_row *,
|
2015-03-19 05:05:37 +08:00
|
|
|
struct sock *, __u16, struct inet_timewait_sock **))
|
2005-12-14 15:25:31 +08:00
|
|
|
{
|
|
|
|
|
struct inet_hashinfo *hinfo = death_row->hashinfo;
|
2009-10-15 14:30:45 +08:00
|
|
|
const unsigned short snum = inet_sk(sk)->inet_num;
|
2007-02-09 22:24:47 +08:00
|
|
|
struct inet_bind_hashbucket *head;
|
|
|
|
|
struct inet_bind_bucket *tb;
|
2005-12-14 15:25:31 +08:00
|
|
|
int ret;
|
2008-03-26 01:26:21 +08:00
|
|
|
struct net *net = sock_net(sk);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2007-02-09 22:24:47 +08:00
|
|
|
if (!snum) {
|
2007-10-11 08:30:46 +08:00
|
|
|
int i, remaining, low, high, port;
|
2005-12-14 15:25:31 +08:00
|
|
|
static u32 hint;
|
2008-02-05 19:14:44 +08:00
|
|
|
u32 offset = hint + port_offset;
|
2007-02-09 22:24:47 +08:00
|
|
|
struct inet_timewait_sock *tw = NULL;
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2013-09-29 05:10:59 +08:00
|
|
|
inet_get_local_port_range(net, &low, &high);
|
2007-10-19 13:00:17 +08:00
|
|
|
remaining = (high - low) + 1;
|
2007-10-11 08:30:46 +08:00
|
|
|
|
tcp/dccp: try to not exhaust ip_local_port_range in connect()
A long standing problem on busy servers is the tiny available TCP port
range (/proc/sys/net/ipv4/ip_local_port_range) and the default
sequential allocation of source ports in connect() system call.
If a host is having a lot of active TCP sessions, chances are
very high that all ports are in use by at least one flow,
and subsequent bind(0) attempts fail, or have to scan a big portion of
space to find a slot.
In this patch, I changed the starting point in __inet_hash_connect()
so that we try to favor even [1] ports, leaving odd ports for bind()
users.
We still perform a sequential search, so there is no guarantee, but
if connect() targets are very different, end result is we leave
more ports available to bind(), and we spread them all over the range,
lowering time for both connect() and bind() to find a slot.
This strategy only works well if /proc/sys/net/ipv4/ip_local_port_range
is even, ie if start/end values have different parity.
Therefore, default /proc/sys/net/ipv4/ip_local_port_range was changed to
32768 - 60999 (instead of 32768 - 61000)
There is no change on security aspects here, only some poor hashing
schemes could be eventually impacted by this change.
[1] : The odd/even property depends on ip_local_port_range values parity
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-25 05:49:35 +08:00
|
|
|
/* By starting with offset being an even number,
|
|
|
|
|
* we tend to leave about 50% of ports for other uses,
|
|
|
|
|
* like bind(0).
|
|
|
|
|
*/
|
|
|
|
|
offset &= ~1;
|
|
|
|
|
|
2007-02-09 22:24:47 +08:00
|
|
|
local_bh_disable();
|
tcp/dccp: try to not exhaust ip_local_port_range in connect()
A long standing problem on busy servers is the tiny available TCP port
range (/proc/sys/net/ipv4/ip_local_port_range) and the default
sequential allocation of source ports in connect() system call.
If a host is having a lot of active TCP sessions, chances are
very high that all ports are in use by at least one flow,
and subsequent bind(0) attempts fail, or have to scan a big portion of
space to find a slot.
In this patch, I changed the starting point in __inet_hash_connect()
so that we try to favor even [1] ports, leaving odd ports for bind()
users.
We still perform a sequential search, so there is no guarantee, but
if connect() targets are very different, end result is we leave
more ports available to bind(), and we spread them all over the range,
lowering time for both connect() and bind() to find a slot.
This strategy only works well if /proc/sys/net/ipv4/ip_local_port_range
is even, ie if start/end values have different parity.
Therefore, default /proc/sys/net/ipv4/ip_local_port_range was changed to
32768 - 60999 (instead of 32768 - 61000)
There is no change on security aspects here, only some poor hashing
schemes could be eventually impacted by this change.
[1] : The odd/even property depends on ip_local_port_range values parity
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-25 05:49:35 +08:00
|
|
|
for (i = 0; i < remaining; i++) {
|
2007-10-11 08:30:46 +08:00
|
|
|
port = low + (i + offset) % remaining;
|
2014-05-13 07:04:53 +08:00
|
|
|
if (inet_is_local_reserved_port(net, port))
|
2010-05-05 08:27:06 +08:00
|
|
|
continue;
|
2008-06-17 08:12:49 +08:00
|
|
|
head = &hinfo->bhash[inet_bhashfn(net, port,
|
|
|
|
|
hinfo->bhash_size)];
|
2007-02-09 22:24:47 +08:00
|
|
|
spin_lock(&head->lock);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2007-02-09 22:24:47 +08:00
|
|
|
/* Does not bother with rcv_saddr checks,
|
|
|
|
|
* because the established check is already
|
|
|
|
|
* unique enough.
|
|
|
|
|
*/
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 09:06:00 +08:00
|
|
|
inet_bind_bucket_for_each(tb, &head->chain) {
|
2009-11-26 07:14:13 +08:00
|
|
|
if (net_eq(ib_net(tb), net) &&
|
|
|
|
|
tb->port == port) {
|
2013-01-22 17:50:24 +08:00
|
|
|
if (tb->fastreuse >= 0 ||
|
|
|
|
|
tb->fastreuseport >= 0)
|
2007-02-09 22:24:47 +08:00
|
|
|
goto next_port;
|
inet: Allowing more than 64k connections and heavily optimize bind(0) time.
With simple extension to the binding mechanism, which allows to bind more
than 64k sockets (or smaller amount, depending on sysctl parameters),
we have to traverse the whole bind hash table to find out empty bucket.
And while it is not a problem for example for 32k connections, bind()
completion time grows exponentially (since after each successful binding
we have to traverse one bucket more to find empty one) even if we start
each time from random offset inside the hash table.
So, when hash table is full, and we want to add another socket, we have
to traverse the whole table no matter what, so effectivelly this will be
the worst case performance and it will be constant.
Attached picture shows bind() time depending on number of already bound
sockets.
Green area corresponds to the usual binding to zero port process, which
turns on kernel port selection as described above. Red area is the bind
process, when number of reuse-bound sockets is not limited by 64k (or
sysctl parameters). The same exponential growth (hidden by the green
area) before number of ports reaches sysctl limit.
At this time bind hash table has exactly one reuse-enbaled socket in a
bucket, but it is possible that they have different addresses. Actually
kernel selects the first port to try randomly, so at the beginning bind
will take roughly constant time, but with time number of port to check
after random start will increase. And that will have exponential growth,
but because of above random selection, not every next port selection
will necessary take longer time than previous. So we have to consider
the area below in the graph (if you could zoom it, you could find, that
there are many different times placed there), so area can hide another.
Blue area corresponds to the port selection optimization.
This is rather simple design approach: hashtable now maintains (unprecise
and racely updated) number of currently bound sockets, and when number
of such sockets becomes greater than predefined value (I use maximum
port range defined by sysctls), we stop traversing the whole bind hash
table and just stop at first matching bucket after random start. Above
limit roughly corresponds to the case, when bind hash table is full and
we turned on mechanism of allowing to bind more reuse-enabled sockets,
so it does not change behaviour of other sockets.
Signed-off-by: Evgeniy Polyakov <zbr@ioremap.net>
Tested-by: Denys Fedoryschenko <denys@visp.net.lb>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-01-20 08:46:02 +08:00
|
|
|
WARN_ON(hlist_empty(&tb->owners));
|
2008-01-31 21:04:45 +08:00
|
|
|
if (!check_established(death_row, sk,
|
|
|
|
|
port, &tw))
|
2007-02-09 22:24:47 +08:00
|
|
|
goto ok;
|
|
|
|
|
goto next_port;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2008-01-31 21:05:50 +08:00
|
|
|
tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
|
|
|
|
|
net, head, port);
|
2007-02-09 22:24:47 +08:00
|
|
|
if (!tb) {
|
|
|
|
|
spin_unlock(&head->lock);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
tb->fastreuse = -1;
|
2013-01-22 17:50:24 +08:00
|
|
|
tb->fastreuseport = -1;
|
2007-02-09 22:24:47 +08:00
|
|
|
goto ok;
|
|
|
|
|
|
|
|
|
|
next_port:
|
|
|
|
|
spin_unlock(&head->lock);
|
|
|
|
|
}
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
|
|
|
|
|
return -EADDRNOTAVAIL;
|
2005-12-14 15:25:31 +08:00
|
|
|
|
|
|
|
|
ok:
|
tcp/dccp: try to not exhaust ip_local_port_range in connect()
A long standing problem on busy servers is the tiny available TCP port
range (/proc/sys/net/ipv4/ip_local_port_range) and the default
sequential allocation of source ports in connect() system call.
If a host is having a lot of active TCP sessions, chances are
very high that all ports are in use by at least one flow,
and subsequent bind(0) attempts fail, or have to scan a big portion of
space to find a slot.
In this patch, I changed the starting point in __inet_hash_connect()
so that we try to favor even [1] ports, leaving odd ports for bind()
users.
We still perform a sequential search, so there is no guarantee, but
if connect() targets are very different, end result is we leave
more ports available to bind(), and we spread them all over the range,
lowering time for both connect() and bind() to find a slot.
This strategy only works well if /proc/sys/net/ipv4/ip_local_port_range
is even, ie if start/end values have different parity.
Therefore, default /proc/sys/net/ipv4/ip_local_port_range was changed to
32768 - 60999 (instead of 32768 - 61000)
There is no change on security aspects here, only some poor hashing
schemes could be eventually impacted by this change.
[1] : The odd/even property depends on ip_local_port_range values parity
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-25 05:49:35 +08:00
|
|
|
hint += (i + 2) & ~1;
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2007-02-09 22:24:47 +08:00
|
|
|
/* Head lock still held and bh's disabled */
|
|
|
|
|
inet_bind_hash(sk, tb, port);
|
2005-12-14 15:25:31 +08:00
|
|
|
if (sk_unhashed(sk)) {
|
2009-10-15 14:30:45 +08:00
|
|
|
inet_sk(sk)->inet_sport = htons(port);
|
2015-10-22 23:20:46 +08:00
|
|
|
inet_ehash_nolisten(sk, (struct sock *)tw);
|
2007-02-09 22:24:47 +08:00
|
|
|
}
|
2009-12-04 11:47:42 +08:00
|
|
|
if (tw)
|
2015-07-09 05:28:29 +08:00
|
|
|
inet_twsk_bind_unhash(tw, hinfo);
|
2007-02-09 22:24:47 +08:00
|
|
|
spin_unlock(&head->lock);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2015-07-09 05:28:30 +08:00
|
|
|
if (tw)
|
|
|
|
|
inet_twsk_deschedule_put(tw);
|
2005-12-14 15:25:31 +08:00
|
|
|
|
|
|
|
|
ret = 0;
|
|
|
|
|
goto out;
|
2007-02-09 22:24:47 +08:00
|
|
|
}
|
2005-12-14 15:25:31 +08:00
|
|
|
|
2008-06-17 08:12:49 +08:00
|
|
|
head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)];
|
2007-02-09 22:24:47 +08:00
|
|
|
tb = inet_csk(sk)->icsk_bind_hash;
|
2005-12-14 15:25:31 +08:00
|
|
|
spin_lock_bh(&head->lock);
|
|
|
|
|
if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
|
2015-10-22 23:20:46 +08:00
|
|
|
inet_ehash_nolisten(sk, NULL);
|
2005-12-14 15:25:31 +08:00
|
|
|
spin_unlock_bh(&head->lock);
|
|
|
|
|
return 0;
|
|
|
|
|
} else {
|
|
|
|
|
spin_unlock(&head->lock);
|
|
|
|
|
/* No definite answer... Walk to established hash table */
|
2008-01-31 21:04:45 +08:00
|
|
|
ret = check_established(death_row, sk, snum, NULL);
|
2005-12-14 15:25:31 +08:00
|
|
|
out:
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
}
|
2008-01-31 21:04:45 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Bind a port for a connect operation and hash it.
|
|
|
|
|
*/
|
|
|
|
|
int inet_hash_connect(struct inet_timewait_death_row *death_row,
|
|
|
|
|
struct sock *sk)
|
|
|
|
|
{
|
2015-05-28 01:46:02 +08:00
|
|
|
u32 port_offset = 0;
|
|
|
|
|
|
|
|
|
|
if (!inet_sk(sk)->inet_num)
|
|
|
|
|
port_offset = inet_sk_port_offset(sk);
|
|
|
|
|
return __inet_hash_connect(death_row, sk, port_offset,
|
2015-03-19 05:05:37 +08:00
|
|
|
__inet_check_established);
|
2008-01-31 21:04:45 +08:00
|
|
|
}
|
2005-12-14 15:25:31 +08:00
|
|
|
EXPORT_SYMBOL_GPL(inet_hash_connect);
|
2008-11-20 16:40:07 +08:00
|
|
|
|
|
|
|
|
void inet_hashinfo_init(struct inet_hashinfo *h)
|
|
|
|
|
{
|
|
|
|
|
int i;
|
|
|
|
|
|
2008-11-24 09:22:55 +08:00
|
|
|
for (i = 0; i < INET_LHTABLE_SIZE; i++) {
|
2008-11-20 16:40:07 +08:00
|
|
|
spin_lock_init(&h->listening_hash[i].lock);
|
2008-11-24 09:22:55 +08:00
|
|
|
INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head,
|
|
|
|
|
i + LISTENING_NULLS_BASE);
|
|
|
|
|
}
|
2008-11-20 16:40:07 +08:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(inet_hashinfo_init);
|
2015-05-26 22:55:34 +08:00
|
|
|
|
|
|
|
|
int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
|
|
|
|
|
{
|
2015-07-22 13:02:00 +08:00
|
|
|
unsigned int locksz = sizeof(spinlock_t);
|
2015-05-26 22:55:34 +08:00
|
|
|
unsigned int i, nblocks = 1;
|
|
|
|
|
|
2015-07-22 13:02:00 +08:00
|
|
|
if (locksz != 0) {
|
2015-05-26 22:55:34 +08:00
|
|
|
/* allocate 2 cache lines or at least one spinlock per cpu */
|
2015-07-22 13:02:00 +08:00
|
|
|
nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U);
|
2015-05-26 22:55:34 +08:00
|
|
|
nblocks = roundup_pow_of_two(nblocks * num_possible_cpus());
|
|
|
|
|
|
|
|
|
|
/* no more locks than number of hash buckets */
|
|
|
|
|
nblocks = min(nblocks, hashinfo->ehash_mask + 1);
|
|
|
|
|
|
2015-07-22 13:02:00 +08:00
|
|
|
hashinfo->ehash_locks = kmalloc_array(nblocks, locksz,
|
2015-05-26 22:55:34 +08:00
|
|
|
GFP_KERNEL | __GFP_NOWARN);
|
|
|
|
|
if (!hashinfo->ehash_locks)
|
2015-07-22 13:02:00 +08:00
|
|
|
hashinfo->ehash_locks = vmalloc(nblocks * locksz);
|
2015-05-26 22:55:34 +08:00
|
|
|
|
|
|
|
|
if (!hashinfo->ehash_locks)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < nblocks; i++)
|
|
|
|
|
spin_lock_init(&hashinfo->ehash_locks[i]);
|
|
|
|
|
}
|
|
|
|
|
hashinfo->ehash_locks_mask = nblocks - 1;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc);
|