2019-05-27 14:55:01 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
*
|
|
|
|
* Implementation of the Transmission Control Protocol(TCP).
|
|
|
|
*
|
|
|
|
* IPv4 specific functions
|
|
|
|
*
|
|
|
|
* code split from:
|
|
|
|
* linux/ipv4/tcp.c
|
|
|
|
* linux/ipv4/tcp_input.c
|
|
|
|
* linux/ipv4/tcp_output.c
|
|
|
|
*
|
|
|
|
* See tcp.c for author information
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Changes:
|
|
|
|
* David S. Miller : New socket lookup architecture.
|
|
|
|
* This code is dedicated to John Dyson.
|
|
|
|
* David S. Miller : Change semantics of established hash,
|
|
|
|
* half is devoted to TIME_WAIT sockets
|
|
|
|
* and the rest go in the other half.
|
|
|
|
* Andi Kleen : Add support for syncookies and fixed
|
|
|
|
* some bugs: ip options weren't passed to
|
|
|
|
* the TCP layer, missed a check for an
|
|
|
|
* ACK bit.
|
|
|
|
* Andi Kleen : Implemented fast path mtu discovery.
|
|
|
|
* Fixed many serious bugs in the
|
2005-06-19 13:47:21 +08:00
|
|
|
* request_sock handling and moved
|
2005-04-17 06:20:36 +08:00
|
|
|
* most of it into the af independent code.
|
|
|
|
* Added tail drop and some other bugfixes.
|
2005-11-11 09:13:47 +08:00
|
|
|
* Added new listen semantics.
|
2005-04-17 06:20:36 +08:00
|
|
|
* Mike McLagan : Routing by source
|
|
|
|
* Juan Jose Ciarlante: ip_dynaddr bits
|
|
|
|
* Andi Kleen: various fixes.
|
|
|
|
* Vitaly E. Lavrov : Transparent proxy revived after year
|
|
|
|
* coma.
|
|
|
|
* Andi Kleen : Fix new listen.
|
|
|
|
* Andi Kleen : Fix accept error reporting.
|
|
|
|
* YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
|
|
|
|
* Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
|
|
|
|
* a single port at the same time.
|
|
|
|
*/
|
|
|
|
|
2012-03-12 15:03:32 +08:00
|
|
|
#define pr_fmt(fmt) "TCP: " fmt
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-12-30 15:04:08 +08:00
|
|
|
#include <linux/bottom_half.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/fcntl.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/random.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/jhash.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/times.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 16:04:11 +08:00
|
|
|
#include <linux/slab.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-09-12 18:01:34 +08:00
|
|
|
#include <net/net_namespace.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <net/icmp.h>
|
2005-08-10 10:59:20 +08:00
|
|
|
#include <net/inet_hashtables.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <net/tcp.h>
|
2005-08-16 13:18:02 +08:00
|
|
|
#include <net/transp_v6.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <net/ipv6.h>
|
|
|
|
#include <net/inet_common.h>
|
2005-12-14 15:25:19 +08:00
|
|
|
#include <net/timewait_sock.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <net/xfrm.h>
|
2011-08-04 11:50:44 +08:00
|
|
|
#include <net/secure_seq.h>
|
2013-07-10 22:13:17 +08:00
|
|
|
#include <net/busy_poll.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#include <linux/inet.h>
|
|
|
|
#include <linux/ipv6.h>
|
|
|
|
#include <linux/stddef.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/seq_file.h>
|
2017-06-16 09:07:06 +08:00
|
|
|
#include <linux/inetdevice.h>
|
2020-07-21 00:34:03 +08:00
|
|
|
#include <linux/btf_ids.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-01-24 21:20:23 +08:00
|
|
|
#include <crypto/hash.h>
|
2006-11-15 11:07:45 +08:00
|
|
|
#include <linux/scatterlist.h>
|
|
|
|
|
2017-10-24 00:20:24 +08:00
|
|
|
#include <trace/events/tcp.h>
|
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2012-01-31 13:18:33 +08:00
|
|
|
static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
|
2011-10-24 14:46:04 +08:00
|
|
|
__be32 daddr, __be32 saddr, const struct tcphdr *th);
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
|
|
|
|
2008-11-20 16:40:07 +08:00
|
|
|
struct inet_hashinfo tcp_hashinfo;
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_hashinfo);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-05-05 21:56:54 +08:00
|
|
|
static u32 tcp_v4_init_seq(const struct sk_buff *skb)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2017-05-05 21:56:54 +08:00
|
|
|
return secure_tcp_seq(ip_hdr(skb)->daddr,
|
|
|
|
ip_hdr(skb)->saddr,
|
|
|
|
tcp_hdr(skb)->dest,
|
|
|
|
tcp_hdr(skb)->source);
|
|
|
|
}
|
|
|
|
|
2017-06-08 01:34:39 +08:00
|
|
|
static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
|
2017-05-05 21:56:54 +08:00
|
|
|
{
|
2017-06-08 01:34:39 +08:00
|
|
|
return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2005-12-14 15:25:19 +08:00
|
|
|
int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
|
|
|
|
{
|
2018-06-04 01:41:17 +08:00
|
|
|
const struct inet_timewait_sock *tw = inet_twsk(sktw);
|
2005-12-14 15:25:19 +08:00
|
|
|
const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
|
|
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
2018-06-04 01:41:17 +08:00
|
|
|
int reuse = sock_net(sk)->ipv4.sysctl_tcp_tw_reuse;
|
|
|
|
|
|
|
|
if (reuse == 2) {
|
|
|
|
/* Still does not detect *everything* that goes through
|
|
|
|
* lo, since we require a loopback src or dst address
|
|
|
|
* or direct binding to 'lo' interface.
|
|
|
|
*/
|
|
|
|
bool loopback = false;
|
|
|
|
if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
|
|
|
|
loopback = true;
|
|
|
|
#if IS_ENABLED(CONFIG_IPV6)
|
|
|
|
if (tw->tw_family == AF_INET6) {
|
|
|
|
if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
|
2019-10-02 01:49:06 +08:00
|
|
|
ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
|
2018-06-04 01:41:17 +08:00
|
|
|
ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
|
2019-10-02 01:49:06 +08:00
|
|
|
ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
|
2018-06-04 01:41:17 +08:00
|
|
|
loopback = true;
|
|
|
|
} else
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
if (ipv4_is_loopback(tw->tw_daddr) ||
|
|
|
|
ipv4_is_loopback(tw->tw_rcv_saddr))
|
|
|
|
loopback = true;
|
|
|
|
}
|
|
|
|
if (!loopback)
|
|
|
|
reuse = 0;
|
|
|
|
}
|
2005-12-14 15:25:19 +08:00
|
|
|
|
|
|
|
/* With PAWS, it is safe from the viewpoint
|
|
|
|
of data integrity. Even without PAWS it is safe provided sequence
|
|
|
|
spaces do not overlap i.e. at data rates <= 80Mbit/sec.
|
|
|
|
|
|
|
|
Actually, the idea is close to VJ's one, only timestamp cache is
|
|
|
|
held not per host, but per port pair and TW bucket is used as state
|
|
|
|
holder.
|
|
|
|
|
|
|
|
If TW bucket has been already destroyed we fall back to VJ's scheme
|
|
|
|
and use initial timestamp retrieved from peer table.
|
|
|
|
*/
|
|
|
|
if (tcptw->tw_ts_recent_stamp &&
|
2018-07-11 18:16:12 +08:00
|
|
|
(!twp || (reuse && time_after32(ktime_get_seconds(),
|
|
|
|
tcptw->tw_ts_recent_stamp)))) {
|
2018-07-11 05:25:20 +08:00
|
|
|
/* In case of repair and re-using TIME-WAIT sockets we still
|
|
|
|
* want to be sure that it is safe as above but honor the
|
|
|
|
* sequence numbers and time stamps set as part of the repair
|
|
|
|
* process.
|
|
|
|
*
|
|
|
|
* Without this check re-using a TIME-WAIT socket with TCP
|
|
|
|
* repair would accumulate a -1 on the repair assigned
|
|
|
|
* sequence number. The first time it is reused the sequence
|
|
|
|
* is -1, the second time -2, etc. This fixes that issue
|
|
|
|
* without appearing to create any others.
|
|
|
|
*/
|
|
|
|
if (likely(!tp->repair)) {
|
2019-10-11 11:17:41 +08:00
|
|
|
u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
|
|
|
|
|
|
|
|
if (!seq)
|
|
|
|
seq = 1;
|
|
|
|
WRITE_ONCE(tp->write_seq, seq);
|
2018-07-11 05:25:20 +08:00
|
|
|
tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
|
|
|
|
tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
|
|
|
|
}
|
2005-12-14 15:25:19 +08:00
|
|
|
sock_hold(sktw);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(tcp_twsk_unique);
|
|
|
|
|
2018-03-31 06:08:05 +08:00
|
|
|
static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
|
|
|
|
int addr_len)
|
|
|
|
{
|
|
|
|
/* This check is replicated from tcp_v4_connect() and intended to
|
|
|
|
* prevent BPF program called below from accessing bytes that are out
|
|
|
|
* of the bound specified by user in addr_len.
|
|
|
|
*/
|
|
|
|
if (addr_len < sizeof(struct sockaddr_in))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
sock_owned_by_me(sk);
|
|
|
|
|
|
|
|
return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* This will initiate an outgoing connection. */
|
|
|
|
int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
|
|
|
|
{
|
2011-04-27 04:28:44 +08:00
|
|
|
struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
2011-02-25 05:38:12 +08:00
|
|
|
__be16 orig_sport, orig_dport;
|
2006-09-27 12:27:15 +08:00
|
|
|
__be32 daddr, nexthop;
|
2011-05-07 07:11:19 +08:00
|
|
|
struct flowi4 *fl4;
|
2011-04-27 04:28:44 +08:00
|
|
|
struct rtable *rt;
|
2005-04-17 06:20:36 +08:00
|
|
|
int err;
|
2011-04-21 17:45:37 +08:00
|
|
|
struct ip_options_rcu *inet_opt;
|
2016-12-28 17:52:32 +08:00
|
|
|
struct inet_timewait_death_row *tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (addr_len < sizeof(struct sockaddr_in))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (usin->sin_family != AF_INET)
|
|
|
|
return -EAFNOSUPPORT;
|
|
|
|
|
|
|
|
nexthop = daddr = usin->sin_addr.s_addr;
|
2011-04-21 17:45:37 +08:00
|
|
|
inet_opt = rcu_dereference_protected(inet->inet_opt,
|
2016-04-05 23:10:15 +08:00
|
|
|
lockdep_sock_is_held(sk));
|
2011-04-21 17:45:37 +08:00
|
|
|
if (inet_opt && inet_opt->opt.srr) {
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!daddr)
|
|
|
|
return -EINVAL;
|
2011-04-21 17:45:37 +08:00
|
|
|
nexthop = inet_opt->opt.faddr;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2011-02-25 05:38:12 +08:00
|
|
|
orig_sport = inet->inet_sport;
|
|
|
|
orig_dport = usin->sin_port;
|
2011-05-07 07:11:19 +08:00
|
|
|
fl4 = &inet->cork.fl.u.ip4;
|
|
|
|
rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
|
2011-03-03 06:31:35 +08:00
|
|
|
RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
|
|
|
|
IPPROTO_TCP,
|
2013-08-28 14:04:14 +08:00
|
|
|
orig_sport, orig_dport, sk);
|
2011-03-03 06:31:35 +08:00
|
|
|
if (IS_ERR(rt)) {
|
|
|
|
err = PTR_ERR(rt);
|
|
|
|
if (err == -ENETUNREACH)
|
2013-11-29 01:51:22 +08:00
|
|
|
IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
|
2011-03-03 06:31:35 +08:00
|
|
|
return err;
|
2007-06-01 13:49:28 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
|
|
|
|
ip_rt_put(rt);
|
|
|
|
return -ENETUNREACH;
|
|
|
|
}
|
|
|
|
|
2011-04-21 17:45:37 +08:00
|
|
|
if (!inet_opt || !inet_opt->opt.srr)
|
2011-05-07 07:11:19 +08:00
|
|
|
daddr = fl4->daddr;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-10-15 14:30:45 +08:00
|
|
|
if (!inet->inet_saddr)
|
2011-05-07 07:11:19 +08:00
|
|
|
inet->inet_saddr = fl4->saddr;
|
2015-03-19 05:05:35 +08:00
|
|
|
sk_rcv_saddr_set(sk, inet->inet_saddr);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-10-15 14:30:45 +08:00
|
|
|
if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Reset inherited state */
|
|
|
|
tp->rx_opt.ts_recent = 0;
|
|
|
|
tp->rx_opt.ts_recent_stamp = 0;
|
2012-04-19 11:40:39 +08:00
|
|
|
if (likely(!tp->repair))
|
2019-10-11 11:17:41 +08:00
|
|
|
WRITE_ONCE(tp->write_seq, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2009-10-15 14:30:45 +08:00
|
|
|
inet->inet_dport = usin->sin_port;
|
2015-03-19 05:05:35 +08:00
|
|
|
sk_daddr_set(sk, daddr);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-12-14 15:26:10 +08:00
|
|
|
inet_csk(sk)->icsk_ext_hdr_len = 0;
|
2011-04-21 17:45:37 +08:00
|
|
|
if (inet_opt)
|
|
|
|
inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-11-10 17:51:18 +08:00
|
|
|
tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* Socket identity is still unknown (sport may be zero).
|
|
|
|
* However we set state to SYN-SENT and not releasing socket
|
|
|
|
* lock select source port, enter ourselves into the hash tables and
|
|
|
|
* complete initialization after this.
|
|
|
|
*/
|
|
|
|
tcp_set_state(sk, TCP_SYN_SENT);
|
2016-12-28 17:52:32 +08:00
|
|
|
err = inet_hash_connect(tcp_death_row, sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (err)
|
|
|
|
goto failure;
|
|
|
|
|
2015-07-29 07:02:05 +08:00
|
|
|
sk_set_txhash(sk);
|
2014-10-23 00:12:01 +08:00
|
|
|
|
2011-05-07 07:11:19 +08:00
|
|
|
rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
|
2011-03-03 06:31:35 +08:00
|
|
|
inet->inet_sport, inet->inet_dport, sk);
|
|
|
|
if (IS_ERR(rt)) {
|
|
|
|
err = PTR_ERR(rt);
|
|
|
|
rt = NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
goto failure;
|
2011-03-03 06:31:35 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
/* OK, now commit destination to socket. */
|
2006-07-01 04:36:35 +08:00
|
|
|
sk->sk_gso_type = SKB_GSO_TCPV4;
|
2010-06-11 14:31:35 +08:00
|
|
|
sk_setup_caps(sk, &rt->dst);
|
net/tcp-fastopen: Add new API support
This patch adds a new socket option, TCP_FASTOPEN_CONNECT, as an
alternative way to perform Fast Open on the active side (client). Prior
to this patch, a client needs to replace the connect() call with
sendto(MSG_FASTOPEN). This can be cumbersome for applications who want
to use Fast Open: these socket operations are often done in lower layer
libraries used by many other applications. Changing these libraries
and/or the socket call sequences are not trivial. A more convenient
approach is to perform Fast Open by simply enabling a socket option when
the socket is created w/o changing other socket calls sequence:
s = socket()
create a new socket
setsockopt(s, IPPROTO_TCP, TCP_FASTOPEN_CONNECT …);
newly introduced sockopt
If set, new functionality described below will be used.
Return ENOTSUPP if TFO is not supported or not enabled in the
kernel.
connect()
With cookie present, return 0 immediately.
With no cookie, initiate 3WHS with TFO cookie-request option and
return -1 with errno = EINPROGRESS.
write()/sendmsg()
With cookie present, send out SYN with data and return the number of
bytes buffered.
With no cookie, and 3WHS not yet completed, return -1 with errno =
EINPROGRESS.
No MSG_FASTOPEN flag is needed.
read()
Return -1 with errno = EWOULDBLOCK/EAGAIN if connect() is called but
write() is not called yet.
Return -1 with errno = EWOULDBLOCK/EAGAIN if connection is
established but no msg is received yet.
Return number of bytes read if socket is established and there is
msg received.
The new API simplifies life for applications that always perform a write()
immediately after a successful connect(). Such applications can now take
advantage of Fast Open by merely making one new setsockopt() call at the time
of creating the socket. Nothing else about the application's socket call
sequence needs to change.
Signed-off-by: Wei Wang <weiwan@google.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-24 02:59:22 +08:00
|
|
|
rt = NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-02-22 18:23:55 +08:00
|
|
|
if (likely(!tp->repair)) {
|
|
|
|
if (!tp->write_seq)
|
2019-10-11 11:17:41 +08:00
|
|
|
WRITE_ONCE(tp->write_seq,
|
|
|
|
secure_tcp_seq(inet->inet_saddr,
|
|
|
|
inet->inet_daddr,
|
|
|
|
inet->inet_sport,
|
|
|
|
usin->sin_port));
|
2017-06-08 01:34:39 +08:00
|
|
|
tp->tsoffset = secure_tcp_ts_off(sock_net(sk),
|
|
|
|
inet->inet_saddr,
|
2017-05-05 21:56:54 +08:00
|
|
|
inet->inet_daddr);
|
2017-02-22 18:23:55 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2019-11-02 01:32:19 +08:00
|
|
|
inet->inet_id = prandom_u32();
|
2005-04-17 06:20:36 +08:00
|
|
|
|
net/tcp-fastopen: Add new API support
This patch adds a new socket option, TCP_FASTOPEN_CONNECT, as an
alternative way to perform Fast Open on the active side (client). Prior
to this patch, a client needs to replace the connect() call with
sendto(MSG_FASTOPEN). This can be cumbersome for applications who want
to use Fast Open: these socket operations are often done in lower layer
libraries used by many other applications. Changing these libraries
and/or the socket call sequences are not trivial. A more convenient
approach is to perform Fast Open by simply enabling a socket option when
the socket is created w/o changing other socket calls sequence:
s = socket()
create a new socket
setsockopt(s, IPPROTO_TCP, TCP_FASTOPEN_CONNECT …);
newly introduced sockopt
If set, new functionality described below will be used.
Return ENOTSUPP if TFO is not supported or not enabled in the
kernel.
connect()
With cookie present, return 0 immediately.
With no cookie, initiate 3WHS with TFO cookie-request option and
return -1 with errno = EINPROGRESS.
write()/sendmsg()
With cookie present, send out SYN with data and return the number of
bytes buffered.
With no cookie, and 3WHS not yet completed, return -1 with errno =
EINPROGRESS.
No MSG_FASTOPEN flag is needed.
read()
Return -1 with errno = EWOULDBLOCK/EAGAIN if connect() is called but
write() is not called yet.
Return -1 with errno = EWOULDBLOCK/EAGAIN if connection is
established but no msg is received yet.
Return number of bytes read if socket is established and there is
msg received.
The new API simplifies life for applications that always perform a write()
immediately after a successful connect(). Such applications can now take
advantage of Fast Open by merely making one new setsockopt() call at the time
of creating the socket. Nothing else about the application's socket call
sequence needs to change.
Signed-off-by: Wei Wang <weiwan@google.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-24 02:59:22 +08:00
|
|
|
if (tcp_fastopen_defer_connect(sk, &err))
|
|
|
|
return err;
|
|
|
|
if (err)
|
|
|
|
goto failure;
|
|
|
|
|
2012-11-22 09:13:58 +08:00
|
|
|
err = tcp_connect(sk);
|
2012-04-19 11:40:39 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (err)
|
|
|
|
goto failure;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
failure:
|
2006-11-17 20:57:30 +08:00
|
|
|
/*
|
|
|
|
* This unhashes the socket and releases the local port,
|
|
|
|
* if necessary.
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
tcp_set_state(sk, TCP_CLOSE);
|
|
|
|
ip_rt_put(rt);
|
|
|
|
sk->sk_route_caps = 0;
|
2009-10-15 14:30:45 +08:00
|
|
|
inet->inet_dport = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
return err;
|
|
|
|
}
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_connect);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
2012-07-23 15:48:52 +08:00
|
|
|
* This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
|
|
|
|
* It can be called through tcp_release_cb() if socket was owned by user
|
|
|
|
* at the time tcp_v4_err() was called to handle ICMP message.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2014-08-15 00:40:05 +08:00
|
|
|
void tcp_v4_mtu_reduced(struct sock *sk)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2017-03-04 06:08:21 +08:00
|
|
|
struct dst_entry *dst;
|
|
|
|
u32 mtu;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-03-04 06:08:21 +08:00
|
|
|
if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
|
|
|
|
return;
|
|
|
|
mtu = tcp_sk(sk)->mtu_info;
|
2012-07-16 18:28:06 +08:00
|
|
|
dst = inet_csk_update_pmtu(sk, mtu);
|
|
|
|
if (!dst)
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
/* Something is about to be wrong... Remember soft error
|
|
|
|
* for the case, if this connection will not able to recover.
|
|
|
|
*/
|
|
|
|
if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
|
|
|
|
sk->sk_err_soft = EMSGSIZE;
|
|
|
|
|
|
|
|
mtu = dst_mtu(dst);
|
|
|
|
|
|
|
|
if (inet->pmtudisc != IP_PMTUDISC_DONT &&
|
2013-11-05 09:24:17 +08:00
|
|
|
ip_sk_accept_pmtu(sk) &&
|
2005-12-14 15:26:10 +08:00
|
|
|
inet_csk(sk)->icsk_pmtu_cookie > mtu) {
|
2005-04-17 06:20:36 +08:00
|
|
|
tcp_sync_mss(sk, mtu);
|
|
|
|
|
|
|
|
/* Resend the TCP packet because it's
|
|
|
|
* clear that the old packet has been
|
|
|
|
* dropped. This is the new "fast" path mtu
|
|
|
|
* discovery.
|
|
|
|
*/
|
|
|
|
tcp_simple_retransmit(sk);
|
|
|
|
} /* else let the usual retransmit timer handle it */
|
|
|
|
}
|
2014-08-15 00:40:05 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_mtu_reduced);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-07-12 12:27:49 +08:00
|
|
|
static void do_redirect(struct sk_buff *skb, struct sock *sk)
|
|
|
|
{
|
|
|
|
struct dst_entry *dst = __sk_dst_check(sk, 0);
|
|
|
|
|
2012-07-12 15:41:25 +08:00
|
|
|
if (dst)
|
2012-07-17 18:29:28 +08:00
|
|
|
dst->ops->redirect(dst, sk, skb);
|
2012-07-12 12:27:49 +08:00
|
|
|
}
|
|
|
|
|
2015-03-23 01:22:22 +08:00
|
|
|
|
|
|
|
/* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
|
2016-02-03 11:31:12 +08:00
|
|
|
void tcp_req_err(struct sock *sk, u32 seq, bool abort)
|
2015-03-23 01:22:22 +08:00
|
|
|
{
|
|
|
|
struct request_sock *req = inet_reqsk(sk);
|
|
|
|
struct net *net = sock_net(sk);
|
|
|
|
|
|
|
|
/* ICMPs are not backlogged, hence we cannot get
|
|
|
|
* an established socket here.
|
|
|
|
*/
|
|
|
|
if (seq != tcp_rsk(req)->snt_isn) {
|
2016-04-28 07:44:39 +08:00
|
|
|
__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
|
2016-02-03 11:31:12 +08:00
|
|
|
} else if (abort) {
|
2015-03-23 01:22:22 +08:00
|
|
|
/*
|
|
|
|
* Still in SYN_RECV, just remove it silently.
|
|
|
|
* There is no good way to pass the error to the newly
|
|
|
|
* created socket, and POSIX does not want network
|
|
|
|
* errors returned from accept().
|
|
|
|
*/
|
2015-03-24 06:00:41 +08:00
|
|
|
inet_csk_reqsk_queue_drop(req->rsk_listener, req);
|
2016-04-01 23:52:20 +08:00
|
|
|
tcp_listendrop(req->rsk_listener);
|
2015-03-23 01:22:22 +08:00
|
|
|
}
|
2015-10-15 02:16:26 +08:00
|
|
|
reqsk_put(req);
|
2015-03-23 01:22:22 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(tcp_req_err);
|
|
|
|
|
2020-05-27 10:48:49 +08:00
|
|
|
/* TCP-LD (RFC 6069) logic */
|
2020-05-28 08:34:58 +08:00
|
|
|
void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
|
2020-05-27 10:48:49 +08:00
|
|
|
{
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
|
|
|
struct sk_buff *skb;
|
|
|
|
s32 remaining;
|
|
|
|
u32 delta_us;
|
|
|
|
|
|
|
|
if (sock_owned_by_user(sk))
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (seq != tp->snd_una || !icsk->icsk_retransmits ||
|
|
|
|
!icsk->icsk_backoff)
|
|
|
|
return;
|
|
|
|
|
|
|
|
skb = tcp_rtx_queue_head(sk);
|
|
|
|
if (WARN_ON_ONCE(!skb))
|
|
|
|
return;
|
|
|
|
|
|
|
|
icsk->icsk_backoff--;
|
|
|
|
icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
|
|
|
|
icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
|
|
|
|
|
|
|
|
tcp_mstamp_refresh(tp);
|
|
|
|
delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
|
|
|
|
remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
|
|
|
|
|
|
|
|
if (remaining > 0) {
|
|
|
|
inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
|
|
|
|
remaining, TCP_RTO_MAX);
|
|
|
|
} else {
|
|
|
|
/* RTO revert clocked out retransmission.
|
|
|
|
* Will retransmit now.
|
|
|
|
*/
|
|
|
|
tcp_retransmit_timer(sk);
|
|
|
|
}
|
|
|
|
}
|
2020-05-28 08:34:58 +08:00
|
|
|
EXPORT_SYMBOL(tcp_ld_RTO_revert);
|
2020-05-27 10:48:49 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* This routine is called by the ICMP module when it gets some
|
|
|
|
* sort of error condition. If err < 0 then the socket should
|
|
|
|
* be closed and the error returned to the user. If err > 0
|
|
|
|
* it's just the icmp type << 8 | icmp code. After adjustment
|
|
|
|
* header points to the first 8 bytes of the tcp header. We need
|
|
|
|
* to find the appropriate port.
|
|
|
|
*
|
|
|
|
* The locking strategy used here is very "optimistic". When
|
|
|
|
* someone else accesses the socket the ICMP is just dropped
|
|
|
|
* and for some paths there is no check at all.
|
|
|
|
* A more general error queue to queue errors for later handling
|
|
|
|
* is probably better.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2020-05-27 10:48:50 +08:00
|
|
|
int tcp_v4_err(struct sk_buff *skb, u32 info)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2020-05-27 10:48:50 +08:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
|
|
|
struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
|
2005-04-17 06:20:36 +08:00
|
|
|
struct tcp_sock *tp;
|
|
|
|
struct inet_sock *inet;
|
2020-05-27 10:48:50 +08:00
|
|
|
const int type = icmp_hdr(skb)->type;
|
|
|
|
const int code = icmp_hdr(skb)->code;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sock *sk;
|
2014-05-12 11:22:12 +08:00
|
|
|
struct request_sock *fastopen;
|
2017-05-17 05:00:14 +08:00
|
|
|
u32 seq, snd_una;
|
2005-04-17 06:20:36 +08:00
|
|
|
int err;
|
2020-05-27 10:48:50 +08:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-03-23 01:22:22 +08:00
|
|
|
sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
|
|
|
|
th->dest, iph->saddr, ntohs(th->source),
|
2020-05-27 10:48:50 +08:00
|
|
|
inet_iif(skb), 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!sk) {
|
2016-04-28 07:44:29 +08:00
|
|
|
__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
|
2018-11-08 19:19:21 +08:00
|
|
|
return -ENOENT;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
if (sk->sk_state == TCP_TIME_WAIT) {
|
2006-10-11 10:41:46 +08:00
|
|
|
inet_twsk_put(inet_twsk(sk));
|
2018-11-08 19:19:21 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2015-03-23 01:22:22 +08:00
|
|
|
seq = ntohl(th->seq);
|
2018-11-08 19:19:21 +08:00
|
|
|
if (sk->sk_state == TCP_NEW_SYN_RECV) {
|
|
|
|
tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
|
|
|
|
type == ICMP_TIME_EXCEEDED ||
|
|
|
|
(type == ICMP_DEST_UNREACH &&
|
|
|
|
(code == ICMP_NET_UNREACH ||
|
|
|
|
code == ICMP_HOST_UNREACH)));
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
bh_lock_sock(sk);
|
|
|
|
/* If too many ICMPs get dropped on busy
|
|
|
|
* servers this needs to be solved differently.
|
2012-07-23 15:48:52 +08:00
|
|
|
* We do take care of PMTU discovery (RFC1191) special case :
|
|
|
|
* we can receive locally generated ICMP messages while socket is held.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2013-01-20 00:10:37 +08:00
|
|
|
if (sock_owned_by_user(sk)) {
|
|
|
|
if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
|
2016-04-28 07:44:39 +08:00
|
|
|
__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
|
2013-01-20 00:10:37 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
if (sk->sk_state == TCP_CLOSE)
|
|
|
|
goto out;
|
|
|
|
|
2010-03-18 19:27:32 +08:00
|
|
|
if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
|
2016-04-28 07:44:39 +08:00
|
|
|
__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
|
2010-03-18 19:27:32 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
tp = tcp_sk(sk);
|
2014-05-12 11:22:12 +08:00
|
|
|
/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
|
2019-10-11 11:17:38 +08:00
|
|
|
fastopen = rcu_dereference(tp->fastopen_rsk);
|
2014-05-12 11:22:12 +08:00
|
|
|
snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (sk->sk_state != TCP_LISTEN &&
|
2014-05-12 11:22:12 +08:00
|
|
|
!between(seq, snd_una, tp->snd_nxt)) {
|
2016-04-28 07:44:39 +08:00
|
|
|
__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
|
2005-04-17 06:20:36 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (type) {
|
2012-07-12 12:27:49 +08:00
|
|
|
case ICMP_REDIRECT:
|
2017-03-10 13:40:33 +08:00
|
|
|
if (!sock_owned_by_user(sk))
|
2020-05-27 10:48:50 +08:00
|
|
|
do_redirect(skb, sk);
|
2012-07-12 12:27:49 +08:00
|
|
|
goto out;
|
2005-04-17 06:20:36 +08:00
|
|
|
case ICMP_SOURCE_QUENCH:
|
|
|
|
/* Just silently ignore these. */
|
|
|
|
goto out;
|
|
|
|
case ICMP_PARAMETERPROB:
|
|
|
|
err = EPROTO;
|
|
|
|
break;
|
|
|
|
case ICMP_DEST_UNREACH:
|
|
|
|
if (code > NR_ICMP_UNREACH)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
|
2013-03-18 15:01:28 +08:00
|
|
|
/* We are not interested in TCP_LISTEN and open_requests
|
|
|
|
* (SYN-ACKs send out by Linux are always <576bytes so
|
|
|
|
* they should go through unfragmented).
|
|
|
|
*/
|
|
|
|
if (sk->sk_state == TCP_LISTEN)
|
|
|
|
goto out;
|
|
|
|
|
2012-07-23 15:48:52 +08:00
|
|
|
tp->mtu_info = info;
|
tcp: fix possible socket refcount problem
Commit 6f458dfb40 (tcp: improve latencies of timer triggered events)
added bug leading to following trace :
[ 2866.131281] IPv4: Attempt to release TCP socket in state 1 ffff880019ec0000
[ 2866.131726]
[ 2866.132188] =========================
[ 2866.132281] [ BUG: held lock freed! ]
[ 2866.132281] 3.6.0-rc1+ #622 Not tainted
[ 2866.132281] -------------------------
[ 2866.132281] kworker/0:1/652 is freeing memory ffff880019ec0000-ffff880019ec0a1f, with a lock still held there!
[ 2866.132281] (sk_lock-AF_INET-RPC){+.+...}, at: [<ffffffff81903619>] tcp_sendmsg+0x29/0xcc6
[ 2866.132281] 4 locks held by kworker/0:1/652:
[ 2866.132281] #0: (rpciod){.+.+.+}, at: [<ffffffff81083567>] process_one_work+0x1de/0x47f
[ 2866.132281] #1: ((&task->u.tk_work)){+.+.+.}, at: [<ffffffff81083567>] process_one_work+0x1de/0x47f
[ 2866.132281] #2: (sk_lock-AF_INET-RPC){+.+...}, at: [<ffffffff81903619>] tcp_sendmsg+0x29/0xcc6
[ 2866.132281] #3: (&icsk->icsk_retransmit_timer){+.-...}, at: [<ffffffff81078017>] run_timer_softirq+0x1ad/0x35f
[ 2866.132281]
[ 2866.132281] stack backtrace:
[ 2866.132281] Pid: 652, comm: kworker/0:1 Not tainted 3.6.0-rc1+ #622
[ 2866.132281] Call Trace:
[ 2866.132281] <IRQ> [<ffffffff810bc527>] debug_check_no_locks_freed+0x112/0x159
[ 2866.132281] [<ffffffff818a0839>] ? __sk_free+0xfd/0x114
[ 2866.132281] [<ffffffff811549fa>] kmem_cache_free+0x6b/0x13a
[ 2866.132281] [<ffffffff818a0839>] __sk_free+0xfd/0x114
[ 2866.132281] [<ffffffff818a08c0>] sk_free+0x1c/0x1e
[ 2866.132281] [<ffffffff81911e1c>] tcp_write_timer+0x51/0x56
[ 2866.132281] [<ffffffff81078082>] run_timer_softirq+0x218/0x35f
[ 2866.132281] [<ffffffff81078017>] ? run_timer_softirq+0x1ad/0x35f
[ 2866.132281] [<ffffffff810f5831>] ? rb_commit+0x58/0x85
[ 2866.132281] [<ffffffff81911dcb>] ? tcp_write_timer_handler+0x148/0x148
[ 2866.132281] [<ffffffff81070bd6>] __do_softirq+0xcb/0x1f9
[ 2866.132281] [<ffffffff81a0a00c>] ? _raw_spin_unlock+0x29/0x2e
[ 2866.132281] [<ffffffff81a1227c>] call_softirq+0x1c/0x30
[ 2866.132281] [<ffffffff81039f38>] do_softirq+0x4a/0xa6
[ 2866.132281] [<ffffffff81070f2b>] irq_exit+0x51/0xad
[ 2866.132281] [<ffffffff81a129cd>] do_IRQ+0x9d/0xb4
[ 2866.132281] [<ffffffff81a0a3ef>] common_interrupt+0x6f/0x6f
[ 2866.132281] <EOI> [<ffffffff8109d006>] ? sched_clock_cpu+0x58/0xd1
[ 2866.132281] [<ffffffff81a0a172>] ? _raw_spin_unlock_irqrestore+0x4c/0x56
[ 2866.132281] [<ffffffff81078692>] mod_timer+0x178/0x1a9
[ 2866.132281] [<ffffffff818a00aa>] sk_reset_timer+0x19/0x26
[ 2866.132281] [<ffffffff8190b2cc>] tcp_rearm_rto+0x99/0xa4
[ 2866.132281] [<ffffffff8190dfba>] tcp_event_new_data_sent+0x6e/0x70
[ 2866.132281] [<ffffffff8190f7ea>] tcp_write_xmit+0x7de/0x8e4
[ 2866.132281] [<ffffffff818a565d>] ? __alloc_skb+0xa0/0x1a1
[ 2866.132281] [<ffffffff8190f952>] __tcp_push_pending_frames+0x2e/0x8a
[ 2866.132281] [<ffffffff81904122>] tcp_sendmsg+0xb32/0xcc6
[ 2866.132281] [<ffffffff819229c2>] inet_sendmsg+0xaa/0xd5
[ 2866.132281] [<ffffffff81922918>] ? inet_autobind+0x5f/0x5f
[ 2866.132281] [<ffffffff810ee7f1>] ? trace_clock_local+0x9/0xb
[ 2866.132281] [<ffffffff8189adab>] sock_sendmsg+0xa3/0xc4
[ 2866.132281] [<ffffffff810f5de6>] ? rb_reserve_next_event+0x26f/0x2d5
[ 2866.132281] [<ffffffff8103e6a9>] ? native_sched_clock+0x29/0x6f
[ 2866.132281] [<ffffffff8103e6f8>] ? sched_clock+0x9/0xd
[ 2866.132281] [<ffffffff810ee7f1>] ? trace_clock_local+0x9/0xb
[ 2866.132281] [<ffffffff8189ae03>] kernel_sendmsg+0x37/0x43
[ 2866.132281] [<ffffffff8199ce49>] xs_send_kvec+0x77/0x80
[ 2866.132281] [<ffffffff8199cec1>] xs_sendpages+0x6f/0x1a0
[ 2866.132281] [<ffffffff8107826d>] ? try_to_del_timer_sync+0x55/0x61
[ 2866.132281] [<ffffffff8199d0d2>] xs_tcp_send_request+0x55/0xf1
[ 2866.132281] [<ffffffff8199bb90>] xprt_transmit+0x89/0x1db
[ 2866.132281] [<ffffffff81999bcd>] ? call_connect+0x3c/0x3c
[ 2866.132281] [<ffffffff81999d92>] call_transmit+0x1c5/0x20e
[ 2866.132281] [<ffffffff819a0d55>] __rpc_execute+0x6f/0x225
[ 2866.132281] [<ffffffff81999bcd>] ? call_connect+0x3c/0x3c
[ 2866.132281] [<ffffffff819a0f33>] rpc_async_schedule+0x28/0x34
[ 2866.132281] [<ffffffff810835d6>] process_one_work+0x24d/0x47f
[ 2866.132281] [<ffffffff81083567>] ? process_one_work+0x1de/0x47f
[ 2866.132281] [<ffffffff819a0f0b>] ? __rpc_execute+0x225/0x225
[ 2866.132281] [<ffffffff81083a6d>] worker_thread+0x236/0x317
[ 2866.132281] [<ffffffff81083837>] ? process_scheduled_works+0x2f/0x2f
[ 2866.132281] [<ffffffff8108b7b8>] kthread+0x9a/0xa2
[ 2866.132281] [<ffffffff81a12184>] kernel_thread_helper+0x4/0x10
[ 2866.132281] [<ffffffff81a0a4b0>] ? retint_restore_args+0x13/0x13
[ 2866.132281] [<ffffffff8108b71e>] ? __init_kthread_worker+0x5a/0x5a
[ 2866.132281] [<ffffffff81a12180>] ? gs_change+0x13/0x13
[ 2866.308506] IPv4: Attempt to release TCP socket in state 1 ffff880019ec0000
[ 2866.309689] =============================================================================
[ 2866.310254] BUG TCP (Not tainted): Object already free
[ 2866.310254] -----------------------------------------------------------------------------
[ 2866.310254]
The bug comes from the fact that timer set in sk_reset_timer() can run
before we actually do the sock_hold(). socket refcount reaches zero and
we free the socket too soon.
timer handler is not allowed to reduce socket refcnt if socket is owned
by the user, or we need to change sk_reset_timer() implementation.
We should take a reference on the socket in case TCP_DELACK_TIMER_DEFERRED
or TCP_DELACK_TIMER_DEFERRED bit are set in tsq_flags
Also fix a typo in tcp_delack_timer(), where TCP_WRITE_TIMER_DEFERRED
was used instead of TCP_DELACK_TIMER_DEFERRED.
For consistency, use same socket refcount change for TCP_MTU_REDUCED_DEFERRED,
even if not fired from a timer.
Reported-by: Fengguang Wu <fengguang.wu@intel.com>
Tested-by: Fengguang Wu <fengguang.wu@intel.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2012-08-20 08:22:46 +08:00
|
|
|
if (!sock_owned_by_user(sk)) {
|
2012-07-23 15:48:52 +08:00
|
|
|
tcp_v4_mtu_reduced(sk);
|
tcp: fix possible socket refcount problem
Commit 6f458dfb40 (tcp: improve latencies of timer triggered events)
added bug leading to following trace :
[ 2866.131281] IPv4: Attempt to release TCP socket in state 1 ffff880019ec0000
[ 2866.131726]
[ 2866.132188] =========================
[ 2866.132281] [ BUG: held lock freed! ]
[ 2866.132281] 3.6.0-rc1+ #622 Not tainted
[ 2866.132281] -------------------------
[ 2866.132281] kworker/0:1/652 is freeing memory ffff880019ec0000-ffff880019ec0a1f, with a lock still held there!
[ 2866.132281] (sk_lock-AF_INET-RPC){+.+...}, at: [<ffffffff81903619>] tcp_sendmsg+0x29/0xcc6
[ 2866.132281] 4 locks held by kworker/0:1/652:
[ 2866.132281] #0: (rpciod){.+.+.+}, at: [<ffffffff81083567>] process_one_work+0x1de/0x47f
[ 2866.132281] #1: ((&task->u.tk_work)){+.+.+.}, at: [<ffffffff81083567>] process_one_work+0x1de/0x47f
[ 2866.132281] #2: (sk_lock-AF_INET-RPC){+.+...}, at: [<ffffffff81903619>] tcp_sendmsg+0x29/0xcc6
[ 2866.132281] #3: (&icsk->icsk_retransmit_timer){+.-...}, at: [<ffffffff81078017>] run_timer_softirq+0x1ad/0x35f
[ 2866.132281]
[ 2866.132281] stack backtrace:
[ 2866.132281] Pid: 652, comm: kworker/0:1 Not tainted 3.6.0-rc1+ #622
[ 2866.132281] Call Trace:
[ 2866.132281] <IRQ> [<ffffffff810bc527>] debug_check_no_locks_freed+0x112/0x159
[ 2866.132281] [<ffffffff818a0839>] ? __sk_free+0xfd/0x114
[ 2866.132281] [<ffffffff811549fa>] kmem_cache_free+0x6b/0x13a
[ 2866.132281] [<ffffffff818a0839>] __sk_free+0xfd/0x114
[ 2866.132281] [<ffffffff818a08c0>] sk_free+0x1c/0x1e
[ 2866.132281] [<ffffffff81911e1c>] tcp_write_timer+0x51/0x56
[ 2866.132281] [<ffffffff81078082>] run_timer_softirq+0x218/0x35f
[ 2866.132281] [<ffffffff81078017>] ? run_timer_softirq+0x1ad/0x35f
[ 2866.132281] [<ffffffff810f5831>] ? rb_commit+0x58/0x85
[ 2866.132281] [<ffffffff81911dcb>] ? tcp_write_timer_handler+0x148/0x148
[ 2866.132281] [<ffffffff81070bd6>] __do_softirq+0xcb/0x1f9
[ 2866.132281] [<ffffffff81a0a00c>] ? _raw_spin_unlock+0x29/0x2e
[ 2866.132281] [<ffffffff81a1227c>] call_softirq+0x1c/0x30
[ 2866.132281] [<ffffffff81039f38>] do_softirq+0x4a/0xa6
[ 2866.132281] [<ffffffff81070f2b>] irq_exit+0x51/0xad
[ 2866.132281] [<ffffffff81a129cd>] do_IRQ+0x9d/0xb4
[ 2866.132281] [<ffffffff81a0a3ef>] common_interrupt+0x6f/0x6f
[ 2866.132281] <EOI> [<ffffffff8109d006>] ? sched_clock_cpu+0x58/0xd1
[ 2866.132281] [<ffffffff81a0a172>] ? _raw_spin_unlock_irqrestore+0x4c/0x56
[ 2866.132281] [<ffffffff81078692>] mod_timer+0x178/0x1a9
[ 2866.132281] [<ffffffff818a00aa>] sk_reset_timer+0x19/0x26
[ 2866.132281] [<ffffffff8190b2cc>] tcp_rearm_rto+0x99/0xa4
[ 2866.132281] [<ffffffff8190dfba>] tcp_event_new_data_sent+0x6e/0x70
[ 2866.132281] [<ffffffff8190f7ea>] tcp_write_xmit+0x7de/0x8e4
[ 2866.132281] [<ffffffff818a565d>] ? __alloc_skb+0xa0/0x1a1
[ 2866.132281] [<ffffffff8190f952>] __tcp_push_pending_frames+0x2e/0x8a
[ 2866.132281] [<ffffffff81904122>] tcp_sendmsg+0xb32/0xcc6
[ 2866.132281] [<ffffffff819229c2>] inet_sendmsg+0xaa/0xd5
[ 2866.132281] [<ffffffff81922918>] ? inet_autobind+0x5f/0x5f
[ 2866.132281] [<ffffffff810ee7f1>] ? trace_clock_local+0x9/0xb
[ 2866.132281] [<ffffffff8189adab>] sock_sendmsg+0xa3/0xc4
[ 2866.132281] [<ffffffff810f5de6>] ? rb_reserve_next_event+0x26f/0x2d5
[ 2866.132281] [<ffffffff8103e6a9>] ? native_sched_clock+0x29/0x6f
[ 2866.132281] [<ffffffff8103e6f8>] ? sched_clock+0x9/0xd
[ 2866.132281] [<ffffffff810ee7f1>] ? trace_clock_local+0x9/0xb
[ 2866.132281] [<ffffffff8189ae03>] kernel_sendmsg+0x37/0x43
[ 2866.132281] [<ffffffff8199ce49>] xs_send_kvec+0x77/0x80
[ 2866.132281] [<ffffffff8199cec1>] xs_sendpages+0x6f/0x1a0
[ 2866.132281] [<ffffffff8107826d>] ? try_to_del_timer_sync+0x55/0x61
[ 2866.132281] [<ffffffff8199d0d2>] xs_tcp_send_request+0x55/0xf1
[ 2866.132281] [<ffffffff8199bb90>] xprt_transmit+0x89/0x1db
[ 2866.132281] [<ffffffff81999bcd>] ? call_connect+0x3c/0x3c
[ 2866.132281] [<ffffffff81999d92>] call_transmit+0x1c5/0x20e
[ 2866.132281] [<ffffffff819a0d55>] __rpc_execute+0x6f/0x225
[ 2866.132281] [<ffffffff81999bcd>] ? call_connect+0x3c/0x3c
[ 2866.132281] [<ffffffff819a0f33>] rpc_async_schedule+0x28/0x34
[ 2866.132281] [<ffffffff810835d6>] process_one_work+0x24d/0x47f
[ 2866.132281] [<ffffffff81083567>] ? process_one_work+0x1de/0x47f
[ 2866.132281] [<ffffffff819a0f0b>] ? __rpc_execute+0x225/0x225
[ 2866.132281] [<ffffffff81083a6d>] worker_thread+0x236/0x317
[ 2866.132281] [<ffffffff81083837>] ? process_scheduled_works+0x2f/0x2f
[ 2866.132281] [<ffffffff8108b7b8>] kthread+0x9a/0xa2
[ 2866.132281] [<ffffffff81a12184>] kernel_thread_helper+0x4/0x10
[ 2866.132281] [<ffffffff81a0a4b0>] ? retint_restore_args+0x13/0x13
[ 2866.132281] [<ffffffff8108b71e>] ? __init_kthread_worker+0x5a/0x5a
[ 2866.132281] [<ffffffff81a12180>] ? gs_change+0x13/0x13
[ 2866.308506] IPv4: Attempt to release TCP socket in state 1 ffff880019ec0000
[ 2866.309689] =============================================================================
[ 2866.310254] BUG TCP (Not tainted): Object already free
[ 2866.310254] -----------------------------------------------------------------------------
[ 2866.310254]
The bug comes from the fact that timer set in sk_reset_timer() can run
before we actually do the sock_hold(). socket refcount reaches zero and
we free the socket too soon.
timer handler is not allowed to reduce socket refcnt if socket is owned
by the user, or we need to change sk_reset_timer() implementation.
We should take a reference on the socket in case TCP_DELACK_TIMER_DEFERRED
or TCP_DELACK_TIMER_DEFERRED bit are set in tsq_flags
Also fix a typo in tcp_delack_timer(), where TCP_WRITE_TIMER_DEFERRED
was used instead of TCP_DELACK_TIMER_DEFERRED.
For consistency, use same socket refcount change for TCP_MTU_REDUCED_DEFERRED,
even if not fired from a timer.
Reported-by: Fengguang Wu <fengguang.wu@intel.com>
Tested-by: Fengguang Wu <fengguang.wu@intel.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2012-08-20 08:22:46 +08:00
|
|
|
} else {
|
2016-12-04 03:14:57 +08:00
|
|
|
if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
|
tcp: fix possible socket refcount problem
Commit 6f458dfb40 (tcp: improve latencies of timer triggered events)
added bug leading to following trace :
[ 2866.131281] IPv4: Attempt to release TCP socket in state 1 ffff880019ec0000
[ 2866.131726]
[ 2866.132188] =========================
[ 2866.132281] [ BUG: held lock freed! ]
[ 2866.132281] 3.6.0-rc1+ #622 Not tainted
[ 2866.132281] -------------------------
[ 2866.132281] kworker/0:1/652 is freeing memory ffff880019ec0000-ffff880019ec0a1f, with a lock still held there!
[ 2866.132281] (sk_lock-AF_INET-RPC){+.+...}, at: [<ffffffff81903619>] tcp_sendmsg+0x29/0xcc6
[ 2866.132281] 4 locks held by kworker/0:1/652:
[ 2866.132281] #0: (rpciod){.+.+.+}, at: [<ffffffff81083567>] process_one_work+0x1de/0x47f
[ 2866.132281] #1: ((&task->u.tk_work)){+.+.+.}, at: [<ffffffff81083567>] process_one_work+0x1de/0x47f
[ 2866.132281] #2: (sk_lock-AF_INET-RPC){+.+...}, at: [<ffffffff81903619>] tcp_sendmsg+0x29/0xcc6
[ 2866.132281] #3: (&icsk->icsk_retransmit_timer){+.-...}, at: [<ffffffff81078017>] run_timer_softirq+0x1ad/0x35f
[ 2866.132281]
[ 2866.132281] stack backtrace:
[ 2866.132281] Pid: 652, comm: kworker/0:1 Not tainted 3.6.0-rc1+ #622
[ 2866.132281] Call Trace:
[ 2866.132281] <IRQ> [<ffffffff810bc527>] debug_check_no_locks_freed+0x112/0x159
[ 2866.132281] [<ffffffff818a0839>] ? __sk_free+0xfd/0x114
[ 2866.132281] [<ffffffff811549fa>] kmem_cache_free+0x6b/0x13a
[ 2866.132281] [<ffffffff818a0839>] __sk_free+0xfd/0x114
[ 2866.132281] [<ffffffff818a08c0>] sk_free+0x1c/0x1e
[ 2866.132281] [<ffffffff81911e1c>] tcp_write_timer+0x51/0x56
[ 2866.132281] [<ffffffff81078082>] run_timer_softirq+0x218/0x35f
[ 2866.132281] [<ffffffff81078017>] ? run_timer_softirq+0x1ad/0x35f
[ 2866.132281] [<ffffffff810f5831>] ? rb_commit+0x58/0x85
[ 2866.132281] [<ffffffff81911dcb>] ? tcp_write_timer_handler+0x148/0x148
[ 2866.132281] [<ffffffff81070bd6>] __do_softirq+0xcb/0x1f9
[ 2866.132281] [<ffffffff81a0a00c>] ? _raw_spin_unlock+0x29/0x2e
[ 2866.132281] [<ffffffff81a1227c>] call_softirq+0x1c/0x30
[ 2866.132281] [<ffffffff81039f38>] do_softirq+0x4a/0xa6
[ 2866.132281] [<ffffffff81070f2b>] irq_exit+0x51/0xad
[ 2866.132281] [<ffffffff81a129cd>] do_IRQ+0x9d/0xb4
[ 2866.132281] [<ffffffff81a0a3ef>] common_interrupt+0x6f/0x6f
[ 2866.132281] <EOI> [<ffffffff8109d006>] ? sched_clock_cpu+0x58/0xd1
[ 2866.132281] [<ffffffff81a0a172>] ? _raw_spin_unlock_irqrestore+0x4c/0x56
[ 2866.132281] [<ffffffff81078692>] mod_timer+0x178/0x1a9
[ 2866.132281] [<ffffffff818a00aa>] sk_reset_timer+0x19/0x26
[ 2866.132281] [<ffffffff8190b2cc>] tcp_rearm_rto+0x99/0xa4
[ 2866.132281] [<ffffffff8190dfba>] tcp_event_new_data_sent+0x6e/0x70
[ 2866.132281] [<ffffffff8190f7ea>] tcp_write_xmit+0x7de/0x8e4
[ 2866.132281] [<ffffffff818a565d>] ? __alloc_skb+0xa0/0x1a1
[ 2866.132281] [<ffffffff8190f952>] __tcp_push_pending_frames+0x2e/0x8a
[ 2866.132281] [<ffffffff81904122>] tcp_sendmsg+0xb32/0xcc6
[ 2866.132281] [<ffffffff819229c2>] inet_sendmsg+0xaa/0xd5
[ 2866.132281] [<ffffffff81922918>] ? inet_autobind+0x5f/0x5f
[ 2866.132281] [<ffffffff810ee7f1>] ? trace_clock_local+0x9/0xb
[ 2866.132281] [<ffffffff8189adab>] sock_sendmsg+0xa3/0xc4
[ 2866.132281] [<ffffffff810f5de6>] ? rb_reserve_next_event+0x26f/0x2d5
[ 2866.132281] [<ffffffff8103e6a9>] ? native_sched_clock+0x29/0x6f
[ 2866.132281] [<ffffffff8103e6f8>] ? sched_clock+0x9/0xd
[ 2866.132281] [<ffffffff810ee7f1>] ? trace_clock_local+0x9/0xb
[ 2866.132281] [<ffffffff8189ae03>] kernel_sendmsg+0x37/0x43
[ 2866.132281] [<ffffffff8199ce49>] xs_send_kvec+0x77/0x80
[ 2866.132281] [<ffffffff8199cec1>] xs_sendpages+0x6f/0x1a0
[ 2866.132281] [<ffffffff8107826d>] ? try_to_del_timer_sync+0x55/0x61
[ 2866.132281] [<ffffffff8199d0d2>] xs_tcp_send_request+0x55/0xf1
[ 2866.132281] [<ffffffff8199bb90>] xprt_transmit+0x89/0x1db
[ 2866.132281] [<ffffffff81999bcd>] ? call_connect+0x3c/0x3c
[ 2866.132281] [<ffffffff81999d92>] call_transmit+0x1c5/0x20e
[ 2866.132281] [<ffffffff819a0d55>] __rpc_execute+0x6f/0x225
[ 2866.132281] [<ffffffff81999bcd>] ? call_connect+0x3c/0x3c
[ 2866.132281] [<ffffffff819a0f33>] rpc_async_schedule+0x28/0x34
[ 2866.132281] [<ffffffff810835d6>] process_one_work+0x24d/0x47f
[ 2866.132281] [<ffffffff81083567>] ? process_one_work+0x1de/0x47f
[ 2866.132281] [<ffffffff819a0f0b>] ? __rpc_execute+0x225/0x225
[ 2866.132281] [<ffffffff81083a6d>] worker_thread+0x236/0x317
[ 2866.132281] [<ffffffff81083837>] ? process_scheduled_works+0x2f/0x2f
[ 2866.132281] [<ffffffff8108b7b8>] kthread+0x9a/0xa2
[ 2866.132281] [<ffffffff81a12184>] kernel_thread_helper+0x4/0x10
[ 2866.132281] [<ffffffff81a0a4b0>] ? retint_restore_args+0x13/0x13
[ 2866.132281] [<ffffffff8108b71e>] ? __init_kthread_worker+0x5a/0x5a
[ 2866.132281] [<ffffffff81a12180>] ? gs_change+0x13/0x13
[ 2866.308506] IPv4: Attempt to release TCP socket in state 1 ffff880019ec0000
[ 2866.309689] =============================================================================
[ 2866.310254] BUG TCP (Not tainted): Object already free
[ 2866.310254] -----------------------------------------------------------------------------
[ 2866.310254]
The bug comes from the fact that timer set in sk_reset_timer() can run
before we actually do the sock_hold(). socket refcount reaches zero and
we free the socket too soon.
timer handler is not allowed to reduce socket refcnt if socket is owned
by the user, or we need to change sk_reset_timer() implementation.
We should take a reference on the socket in case TCP_DELACK_TIMER_DEFERRED
or TCP_DELACK_TIMER_DEFERRED bit are set in tsq_flags
Also fix a typo in tcp_delack_timer(), where TCP_WRITE_TIMER_DEFERRED
was used instead of TCP_DELACK_TIMER_DEFERRED.
For consistency, use same socket refcount change for TCP_MTU_REDUCED_DEFERRED,
even if not fired from a timer.
Reported-by: Fengguang Wu <fengguang.wu@intel.com>
Tested-by: Fengguang Wu <fengguang.wu@intel.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2012-08-20 08:22:46 +08:00
|
|
|
sock_hold(sk);
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = icmp_err_convert[code].errno;
|
2020-05-27 10:48:49 +08:00
|
|
|
/* check if this ICMP message allows revert of backoff.
|
|
|
|
* (see RFC 6069)
|
|
|
|
*/
|
|
|
|
if (!fastopen &&
|
|
|
|
(code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
|
|
|
|
tcp_ld_RTO_revert(sk, seq);
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
case ICMP_TIME_EXCEEDED:
|
|
|
|
err = EHOSTUNREACH;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (sk->sk_state) {
|
|
|
|
case TCP_SYN_SENT:
|
2014-05-12 11:22:12 +08:00
|
|
|
case TCP_SYN_RECV:
|
|
|
|
/* Only in fast or simultaneous open. If a fast open socket is
|
2020-08-23 07:31:41 +08:00
|
|
|
* already accepted it is treated as a connected one below.
|
2014-05-12 11:22:12 +08:00
|
|
|
*/
|
2015-04-03 16:17:26 +08:00
|
|
|
if (fastopen && !fastopen->sk)
|
2014-05-12 11:22:12 +08:00
|
|
|
break;
|
|
|
|
|
2020-05-27 10:48:50 +08:00
|
|
|
ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
|
tcp: allow traceroute -Mtcp for unpriv users
Unpriv users can use traceroute over plain UDP sockets, but not TCP ones.
$ traceroute -Mtcp 8.8.8.8
You do not have enough privileges to use this traceroute method.
$ traceroute -n -Mudp 8.8.8.8
traceroute to 8.8.8.8 (8.8.8.8), 30 hops max, 60 byte packets
1 192.168.86.1 3.631 ms 3.512 ms 3.405 ms
2 10.1.10.1 4.183 ms 4.125 ms 4.072 ms
3 96.120.88.125 20.621 ms 19.462 ms 20.553 ms
4 96.110.177.65 24.271 ms 25.351 ms 25.250 ms
5 69.139.199.197 44.492 ms 43.075 ms 44.346 ms
6 68.86.143.93 27.969 ms 25.184 ms 25.092 ms
7 96.112.146.18 25.323 ms 96.112.146.22 25.583 ms 96.112.146.26 24.502 ms
8 72.14.239.204 24.405 ms 74.125.37.224 16.326 ms 17.194 ms
9 209.85.251.9 18.154 ms 209.85.247.55 14.449 ms 209.85.251.9 26.296 ms^C
We can easily support traceroute over TCP, by queueing an error message
into socket error queue.
Note that applications need to set IP_RECVERR/IPV6_RECVERR option to
enable this feature, and that the error message is only queued
while in SYN_SNT state.
socket(AF_INET6, SOCK_STREAM, IPPROTO_IP) = 3
setsockopt(3, SOL_IPV6, IPV6_RECVERR, [1], 4) = 0
setsockopt(3, SOL_SOCKET, SO_TIMESTAMP_OLD, [1], 4) = 0
setsockopt(3, SOL_IPV6, IPV6_UNICAST_HOPS, [5], 4) = 0
connect(3, {sa_family=AF_INET6, sin6_port=htons(8787), sin6_flowinfo=htonl(0),
inet_pton(AF_INET6, "2002:a05:6608:297::", &sin6_addr), sin6_scope_id=0}, 28) = -1 EHOSTUNREACH (No route to host)
recvmsg(3, {msg_name={sa_family=AF_INET6, sin6_port=htons(8787), sin6_flowinfo=htonl(0),
inet_pton(AF_INET6, "2002:a05:6608:297::", &sin6_addr), sin6_scope_id=0},
msg_namelen=1024->28, msg_iov=[{iov_base="`\r\337\320\0004\6\1&\7\370\260\200\231\16\27\0\0\0\0\0\0\0\0 \2\n\5f\10\2\227"..., iov_len=1024}],
msg_iovlen=1, msg_control=[{cmsg_len=32, cmsg_level=SOL_SOCKET, cmsg_type=SO_TIMESTAMP_OLD, cmsg_data={tv_sec=1590340680, tv_usec=272424}},
{cmsg_len=60, cmsg_level=SOL_IPV6, cmsg_type=IPV6_RECVERR}],
msg_controllen=96, msg_flags=MSG_ERRQUEUE}, MSG_ERRQUEUE) = 144
Suggested-by: Maciej Żenczykowski <maze@google.com
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Willem de Bruijn <willemb@google.com>
Reviewed-by: Maciej Żenczykowski <maze@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-05-25 02:00:02 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!sock_owned_by_user(sk)) {
|
|
|
|
sk->sk_err = err;
|
|
|
|
|
|
|
|
sk->sk_error_report(sk);
|
|
|
|
|
|
|
|
tcp_done(sk);
|
|
|
|
} else {
|
|
|
|
sk->sk_err_soft = err;
|
|
|
|
}
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If we've already connected we will keep trying
|
|
|
|
* until we time out, or the user gives up.
|
|
|
|
*
|
|
|
|
* rfc1122 4.2.3.9 allows to consider as hard errors
|
|
|
|
* only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
|
|
|
|
* but it is obsoleted by pmtu discovery).
|
|
|
|
*
|
|
|
|
* Note, that in modern internet, where routing is unreliable
|
|
|
|
* and in each dark corner broken firewalls sit, sending random
|
|
|
|
* errors ordered by their masters even this two messages finally lose
|
|
|
|
* their original sense (even Linux sends invalid PORT_UNREACHs)
|
|
|
|
*
|
|
|
|
* Now we are in compliance with RFCs.
|
|
|
|
* --ANK (980905)
|
|
|
|
*/
|
|
|
|
|
|
|
|
inet = inet_sk(sk);
|
|
|
|
if (!sock_owned_by_user(sk) && inet->recverr) {
|
|
|
|
sk->sk_err = err;
|
|
|
|
sk->sk_error_report(sk);
|
|
|
|
} else { /* Only an error on timeout */
|
|
|
|
sk->sk_err_soft = err;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
bh_unlock_sock(sk);
|
|
|
|
sock_put(sk);
|
2018-11-08 19:19:21 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2013-06-07 13:11:46 +08:00
|
|
|
void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2007-04-11 12:04:22 +08:00
|
|
|
struct tcphdr *th = tcp_hdr(skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-02-20 03:56:52 +08:00
|
|
|
th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
|
|
|
|
skb->csum_start = skb_transport_header(skb) - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct tcphdr, check);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2010-04-11 10:15:53 +08:00
|
|
|
/* This routine computes an IPv4 TCP checksum. */
|
2010-04-11 10:15:55 +08:00
|
|
|
void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
|
2010-04-11 10:15:53 +08:00
|
|
|
{
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct inet_sock *inet = inet_sk(sk);
|
2010-04-11 10:15:53 +08:00
|
|
|
|
|
|
|
__tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
|
|
|
|
}
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_send_check);
|
2010-04-11 10:15:53 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* This routine will send an RST to the other tcp.
|
|
|
|
*
|
|
|
|
* Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
|
|
|
|
* for reset.
|
|
|
|
* Answer: if a packet caused RST, it is not for a socket
|
|
|
|
* existing in our system, if it is matched to a socket,
|
|
|
|
* it is just duplicate segment or bug in other side's TCP.
|
|
|
|
* So that we build reply only basing on parameters
|
|
|
|
* arrived with segment.
|
|
|
|
* Exception: precedence violation. We do not implement it in any case.
|
|
|
|
*/
|
|
|
|
|
2015-09-29 22:42:39 +08:00
|
|
|
static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct tcphdr *th = tcp_hdr(skb);
|
2006-11-15 11:07:45 +08:00
|
|
|
struct {
|
|
|
|
struct tcphdr th;
|
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2006-11-15 12:51:49 +08:00
|
|
|
__be32 opt[(TCPOLEN_MD5SIG_ALIGNED >> 2)];
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
|
|
|
} rep;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct ip_reply_arg arg;
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2015-12-22 04:29:25 +08:00
|
|
|
struct tcp_md5sig_key *key = NULL;
|
2012-02-01 06:35:48 +08:00
|
|
|
const __u8 *hash_location = NULL;
|
|
|
|
unsigned char newhash[16];
|
|
|
|
int genhash;
|
|
|
|
struct sock *sk1 = NULL;
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
2019-06-14 12:22:35 +08:00
|
|
|
u64 transmit_time = 0;
|
2018-05-10 14:53:51 +08:00
|
|
|
struct sock *ctl_sk;
|
2019-06-14 12:22:35 +08:00
|
|
|
struct net *net;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* Never send a reset in response to a reset. */
|
|
|
|
if (th->rst)
|
|
|
|
return;
|
|
|
|
|
2014-11-25 23:40:04 +08:00
|
|
|
/* If sk not NULL, it means we did a successful lookup and incoming
|
|
|
|
* route had to be correct. prequeue might have dropped our dst.
|
|
|
|
*/
|
|
|
|
if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
/* Swap the send and the receive. */
|
2006-11-15 11:07:45 +08:00
|
|
|
memset(&rep, 0, sizeof(rep));
|
|
|
|
rep.th.dest = th->source;
|
|
|
|
rep.th.source = th->dest;
|
|
|
|
rep.th.doff = sizeof(struct tcphdr) / 4;
|
|
|
|
rep.th.rst = 1;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (th->ack) {
|
2006-11-15 11:07:45 +08:00
|
|
|
rep.th.seq = th->ack_seq;
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
2006-11-15 11:07:45 +08:00
|
|
|
rep.th.ack = 1;
|
|
|
|
rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
|
|
|
|
skb->len - (th->doff << 2));
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2006-11-17 20:57:30 +08:00
|
|
|
memset(&arg, 0, sizeof(arg));
|
2006-11-15 11:07:45 +08:00
|
|
|
arg.iov[0].iov_base = (unsigned char *)&rep;
|
|
|
|
arg.iov[0].iov_len = sizeof(rep.th);
|
|
|
|
|
2014-12-10 01:56:08 +08:00
|
|
|
net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2016-04-01 23:52:17 +08:00
|
|
|
rcu_read_lock();
|
2012-02-01 06:35:48 +08:00
|
|
|
hash_location = tcp_parse_md5sig_option(th);
|
tcp: honour SO_BINDTODEVICE for TW_RST case too
Hannes points out that when we generate tcp reset for timewait sockets we
pretend we found no socket and pass NULL sk to tcp_vX_send_reset().
Make it cope with inet tw sockets and then provide tw sk.
This makes RSTs appear on correct interface when SO_BINDTODEVICE is used.
Packetdrill test case:
// want default route to be used, we rely on BINDTODEVICE
`ip route del 192.0.2.0/24 via 192.168.0.2 dev tun0`
0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
// test case still works due to BINDTODEVICE
0.001 setsockopt(3, SOL_SOCKET, SO_BINDTODEVICE, "tun0", 4) = 0
0.100...0.200 connect(3, ..., ...) = 0
0.100 > S 0:0(0) <mss 1460,sackOK,nop,nop>
0.200 < S. 0:0(0) ack 1 win 32792 <mss 1460,sackOK,nop,nop>
0.200 > . 1:1(0) ack 1
0.210 close(3) = 0
0.210 > F. 1:1(0) ack 1 win 29200
0.300 < . 1:1(0) ack 2 win 46
// more data while in FIN_WAIT2, expect RST
1.300 < P. 1:1001(1000) ack 1 win 46
// fails without this change -- default route is used
1.301 > R 1:1(0) win 0
Reported-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Florian Westphal <fw@strlen.de>
Acked-by: Eric Dumazet <edumazet@google.com>
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-22 04:29:26 +08:00
|
|
|
if (sk && sk_fullsock(sk)) {
|
2019-12-31 06:14:25 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2019-12-31 06:14:28 +08:00
|
|
|
int l3index;
|
2019-12-31 06:14:25 +08:00
|
|
|
|
2019-12-31 06:14:28 +08:00
|
|
|
/* sdif set, means packet ingressed via a device
|
|
|
|
* in an L3 domain and inet_iif is set to it.
|
|
|
|
*/
|
|
|
|
l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
|
2019-12-31 06:14:25 +08:00
|
|
|
addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
2019-12-31 06:14:28 +08:00
|
|
|
key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
2015-12-22 04:29:25 +08:00
|
|
|
} else if (hash_location) {
|
2019-12-31 06:14:25 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2019-12-31 06:14:27 +08:00
|
|
|
int sdif = tcp_v4_sdif(skb);
|
|
|
|
int dif = inet_iif(skb);
|
2019-12-31 06:14:28 +08:00
|
|
|
int l3index;
|
2019-12-31 06:14:25 +08:00
|
|
|
|
2012-02-01 06:35:48 +08:00
|
|
|
/*
|
|
|
|
* active side is lost. Try to find listening socket through
|
|
|
|
* source port, and then find md5 key through listening socket.
|
|
|
|
* we are not loose security here:
|
|
|
|
* Incoming packet is checked with md5 hash with finding key,
|
|
|
|
* no RST generated if md5 hash doesn't match.
|
|
|
|
*/
|
2016-02-11 00:50:38 +08:00
|
|
|
sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
|
|
|
|
ip_hdr(skb)->saddr,
|
2013-01-22 17:50:24 +08:00
|
|
|
th->source, ip_hdr(skb)->daddr,
|
2019-12-31 06:14:27 +08:00
|
|
|
ntohs(th->source), dif, sdif);
|
2012-02-01 06:35:48 +08:00
|
|
|
/* don't send rst if it can't find key */
|
|
|
|
if (!sk1)
|
2016-04-01 23:52:17 +08:00
|
|
|
goto out;
|
|
|
|
|
2019-12-31 06:14:28 +08:00
|
|
|
/* sdif set, means packet ingressed via a device
|
|
|
|
* in an L3 domain and dif is set to it.
|
|
|
|
*/
|
|
|
|
l3index = sdif ? dif : 0;
|
2019-12-31 06:14:25 +08:00
|
|
|
addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
2019-12-31 06:14:28 +08:00
|
|
|
key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
|
2012-02-01 06:35:48 +08:00
|
|
|
if (!key)
|
2016-04-01 23:52:17 +08:00
|
|
|
goto out;
|
|
|
|
|
2012-02-01 06:35:48 +08:00
|
|
|
|
2015-03-25 06:58:55 +08:00
|
|
|
genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
|
2012-02-01 06:35:48 +08:00
|
|
|
if (genhash || memcmp(hash_location, newhash, 16) != 0)
|
2016-04-01 23:52:17 +08:00
|
|
|
goto out;
|
|
|
|
|
2012-02-01 06:35:48 +08:00
|
|
|
}
|
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
if (key) {
|
|
|
|
rep.opt[0] = htonl((TCPOPT_NOP << 24) |
|
|
|
|
(TCPOPT_NOP << 16) |
|
|
|
|
(TCPOPT_MD5SIG << 8) |
|
|
|
|
TCPOLEN_MD5SIG);
|
|
|
|
/* Update length and the length the header thinks exists */
|
|
|
|
arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
|
|
|
|
rep.th.doff = arg.iov[0].iov_len / 4;
|
|
|
|
|
2008-07-19 15:01:42 +08:00
|
|
|
tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
|
2008-10-10 05:37:47 +08:00
|
|
|
key, ip_hdr(skb)->saddr,
|
|
|
|
ip_hdr(skb)->daddr, &rep.th);
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
#endif
|
2007-04-21 13:47:35 +08:00
|
|
|
arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
|
|
|
|
ip_hdr(skb)->saddr, /* XXX */
|
2008-10-09 02:34:06 +08:00
|
|
|
arg.iov[0].iov_len, IPPROTO_TCP, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
arg.csumoffset = offsetof(struct tcphdr, check) / 2;
|
tcp: honour SO_BINDTODEVICE for TW_RST case too
Hannes points out that when we generate tcp reset for timewait sockets we
pretend we found no socket and pass NULL sk to tcp_vX_send_reset().
Make it cope with inet tw sockets and then provide tw sk.
This makes RSTs appear on correct interface when SO_BINDTODEVICE is used.
Packetdrill test case:
// want default route to be used, we rely on BINDTODEVICE
`ip route del 192.0.2.0/24 via 192.168.0.2 dev tun0`
0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
// test case still works due to BINDTODEVICE
0.001 setsockopt(3, SOL_SOCKET, SO_BINDTODEVICE, "tun0", 4) = 0
0.100...0.200 connect(3, ..., ...) = 0
0.100 > S 0:0(0) <mss 1460,sackOK,nop,nop>
0.200 < S. 0:0(0) ack 1 win 32792 <mss 1460,sackOK,nop,nop>
0.200 > . 1:1(0) ack 1
0.210 close(3) = 0
0.210 > F. 1:1(0) ack 1 win 29200
0.300 < . 1:1(0) ack 2 win 46
// more data while in FIN_WAIT2, expect RST
1.300 < P. 1:1001(1000) ack 1 win 46
// fails without this change -- default route is used
1.301 > R 1:1(0) win 0
Reported-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Florian Westphal <fw@strlen.de>
Acked-by: Eric Dumazet <edumazet@google.com>
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-22 04:29:26 +08:00
|
|
|
arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
|
|
|
|
|
2012-02-04 20:38:09 +08:00
|
|
|
/* When socket is gone, all binding information is lost.
|
2012-10-12 12:34:17 +08:00
|
|
|
* routing might fail in this case. No choice here, if we choose to force
|
|
|
|
* input interface, we will misroute in case of asymmetric route.
|
2012-02-04 20:38:09 +08:00
|
|
|
*/
|
2017-10-24 00:20:24 +08:00
|
|
|
if (sk) {
|
2012-10-12 12:34:17 +08:00
|
|
|
arg.bound_dev_if = sk->sk_bound_dev_if;
|
2018-02-07 12:50:23 +08:00
|
|
|
if (sk_fullsock(sk))
|
|
|
|
trace_tcp_send_reset(sk, skb);
|
2017-10-24 00:20:24 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
tcp: honour SO_BINDTODEVICE for TW_RST case too
Hannes points out that when we generate tcp reset for timewait sockets we
pretend we found no socket and pass NULL sk to tcp_vX_send_reset().
Make it cope with inet tw sockets and then provide tw sk.
This makes RSTs appear on correct interface when SO_BINDTODEVICE is used.
Packetdrill test case:
// want default route to be used, we rely on BINDTODEVICE
`ip route del 192.0.2.0/24 via 192.168.0.2 dev tun0`
0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
// test case still works due to BINDTODEVICE
0.001 setsockopt(3, SOL_SOCKET, SO_BINDTODEVICE, "tun0", 4) = 0
0.100...0.200 connect(3, ..., ...) = 0
0.100 > S 0:0(0) <mss 1460,sackOK,nop,nop>
0.200 < S. 0:0(0) ack 1 win 32792 <mss 1460,sackOK,nop,nop>
0.200 > . 1:1(0) ack 1
0.210 close(3) = 0
0.210 > F. 1:1(0) ack 1 win 29200
0.300 < . 1:1(0) ack 2 win 46
// more data while in FIN_WAIT2, expect RST
1.300 < P. 1:1001(1000) ack 1 win 46
// fails without this change -- default route is used
1.301 > R 1:1(0) win 0
Reported-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Florian Westphal <fw@strlen.de>
Acked-by: Eric Dumazet <edumazet@google.com>
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-22 04:29:26 +08:00
|
|
|
BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
|
|
|
|
offsetof(struct inet_timewait_sock, tw_bound_dev_if));
|
|
|
|
|
2011-10-24 15:06:21 +08:00
|
|
|
arg.tos = ip_hdr(skb)->tos;
|
2016-11-04 01:23:43 +08:00
|
|
|
arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
|
2016-05-07 00:46:18 +08:00
|
|
|
local_bh_disable();
|
2019-06-01 10:17:33 +08:00
|
|
|
ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
|
tcp: add optional per socket transmit delay
Adding delays to TCP flows is crucial for studying behavior
of TCP stacks, including congestion control modules.
Linux offers netem module, but it has unpractical constraints :
- Need root access to change qdisc
- Hard to setup on egress if combined with non trivial qdisc like FQ
- Single delay for all flows.
EDT (Earliest Departure Time) adoption in TCP stack allows us
to enable a per socket delay at a very small cost.
Networking tools can now establish thousands of flows, each of them
with a different delay, simulating real world conditions.
This requires FQ packet scheduler or a EDT-enabled NIC.
This patchs adds TCP_TX_DELAY socket option, to set a delay in
usec units.
unsigned int tx_delay = 10000; /* 10 msec */
setsockopt(fd, SOL_TCP, TCP_TX_DELAY, &tx_delay, sizeof(tx_delay));
Note that FQ packet scheduler limits might need some tweaking :
man tc-fq
PARAMETERS
limit
Hard limit on the real queue size. When this limit is
reached, new packets are dropped. If the value is lowered,
packets are dropped so that the new limit is met. Default
is 10000 packets.
flow_limit
Hard limit on the maximum number of packets queued per
flow. Default value is 100.
Use of TCP_TX_DELAY option will increase number of skbs in FQ qdisc,
so packets would be dropped if any of the previous limit is hit.
Use of a jump label makes this support runtime-free, for hosts
never using the option.
Also note that TSQ (TCP Small Queues) limits are slightly changed
with this patch : we need to account that skbs artificially delayed
wont stop us providind more skbs to feed the pipe (netem uses
skb_orphan_partial() for this purpose, but FQ can not use this trick)
Because of that, using big delays might very well trigger
old bugs in TSO auto defer logic and/or sndbuf limited detection.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-13 02:57:25 +08:00
|
|
|
if (sk) {
|
2018-05-10 14:53:51 +08:00
|
|
|
ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
|
|
|
|
inet_twsk(sk)->tw_mark : sk->sk_mark;
|
2019-09-24 23:01:16 +08:00
|
|
|
ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
|
|
|
|
inet_twsk(sk)->tw_priority : sk->sk_priority;
|
2019-06-14 12:22:35 +08:00
|
|
|
transmit_time = tcp_transmit_time(sk);
|
tcp: add optional per socket transmit delay
Adding delays to TCP flows is crucial for studying behavior
of TCP stacks, including congestion control modules.
Linux offers netem module, but it has unpractical constraints :
- Need root access to change qdisc
- Hard to setup on egress if combined with non trivial qdisc like FQ
- Single delay for all flows.
EDT (Earliest Departure Time) adoption in TCP stack allows us
to enable a per socket delay at a very small cost.
Networking tools can now establish thousands of flows, each of them
with a different delay, simulating real world conditions.
This requires FQ packet scheduler or a EDT-enabled NIC.
This patchs adds TCP_TX_DELAY socket option, to set a delay in
usec units.
unsigned int tx_delay = 10000; /* 10 msec */
setsockopt(fd, SOL_TCP, TCP_TX_DELAY, &tx_delay, sizeof(tx_delay));
Note that FQ packet scheduler limits might need some tweaking :
man tc-fq
PARAMETERS
limit
Hard limit on the real queue size. When this limit is
reached, new packets are dropped. If the value is lowered,
packets are dropped so that the new limit is met. Default
is 10000 packets.
flow_limit
Hard limit on the maximum number of packets queued per
flow. Default value is 100.
Use of TCP_TX_DELAY option will increase number of skbs in FQ qdisc,
so packets would be dropped if any of the previous limit is hit.
Use of a jump label makes this support runtime-free, for hosts
never using the option.
Also note that TSQ (TCP Small Queues) limits are slightly changed
with this patch : we need to account that skbs artificially delayed
wont stop us providind more skbs to feed the pipe (netem uses
skb_orphan_partial() for this purpose, but FQ can not use this trick)
Because of that, using big delays might very well trigger
old bugs in TSO auto defer logic and/or sndbuf limited detection.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-13 02:57:25 +08:00
|
|
|
}
|
2018-05-10 14:53:51 +08:00
|
|
|
ip_send_unicast_reply(ctl_sk,
|
2015-01-30 13:35:05 +08:00
|
|
|
skb, &TCP_SKB_CB(skb)->header.h4.opt,
|
2014-09-28 00:50:55 +08:00
|
|
|
ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
|
2019-06-14 12:22:35 +08:00
|
|
|
&arg, arg.iov[0].iov_len,
|
|
|
|
transmit_time);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-05-10 14:53:51 +08:00
|
|
|
ctl_sk->sk_mark = 0;
|
2016-04-28 07:44:32 +08:00
|
|
|
__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
|
|
|
|
__TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
|
2016-05-07 00:46:18 +08:00
|
|
|
local_bh_enable();
|
2012-02-01 06:35:48 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2016-04-01 23:52:17 +08:00
|
|
|
out:
|
|
|
|
rcu_read_unlock();
|
2012-02-01 06:35:48 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
|
|
|
|
outside socket context is ugly, certainly. What can I do?
|
|
|
|
*/
|
|
|
|
|
2016-11-04 01:23:43 +08:00
|
|
|
static void tcp_v4_send_ack(const struct sock *sk,
|
2016-01-22 00:02:54 +08:00
|
|
|
struct sk_buff *skb, u32 seq, u32 ack,
|
2013-02-11 13:50:19 +08:00
|
|
|
u32 win, u32 tsval, u32 tsecr, int oif,
|
2008-10-01 22:41:00 +08:00
|
|
|
struct tcp_md5sig_key *key,
|
2011-10-24 15:06:21 +08:00
|
|
|
int reply_flags, u8 tos)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct tcphdr *th = tcp_hdr(skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
struct {
|
|
|
|
struct tcphdr th;
|
2006-11-15 12:51:49 +08:00
|
|
|
__be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2006-11-15 12:51:49 +08:00
|
|
|
+ (TCPOLEN_MD5SIG_ALIGNED >> 2)
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
|
|
|
];
|
2005-04-17 06:20:36 +08:00
|
|
|
} rep;
|
2016-11-04 01:23:43 +08:00
|
|
|
struct net *net = sock_net(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
struct ip_reply_arg arg;
|
2018-05-10 14:53:51 +08:00
|
|
|
struct sock *ctl_sk;
|
2019-06-14 12:22:35 +08:00
|
|
|
u64 transmit_time;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
memset(&rep.th, 0, sizeof(struct tcphdr));
|
2006-11-17 20:57:30 +08:00
|
|
|
memset(&arg, 0, sizeof(arg));
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
arg.iov[0].iov_base = (unsigned char *)&rep;
|
|
|
|
arg.iov[0].iov_len = sizeof(rep.th);
|
2013-02-11 13:50:19 +08:00
|
|
|
if (tsecr) {
|
2006-11-15 11:07:45 +08:00
|
|
|
rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
|
|
|
|
(TCPOPT_TIMESTAMP << 8) |
|
|
|
|
TCPOLEN_TIMESTAMP);
|
2013-02-11 13:50:19 +08:00
|
|
|
rep.opt[1] = htonl(tsval);
|
|
|
|
rep.opt[2] = htonl(tsecr);
|
2007-01-09 16:11:15 +08:00
|
|
|
arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Swap the send and the receive. */
|
|
|
|
rep.th.dest = th->source;
|
|
|
|
rep.th.source = th->dest;
|
|
|
|
rep.th.doff = arg.iov[0].iov_len / 4;
|
|
|
|
rep.th.seq = htonl(seq);
|
|
|
|
rep.th.ack_seq = htonl(ack);
|
|
|
|
rep.th.ack = 1;
|
|
|
|
rep.th.window = htons(win);
|
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
|
|
|
if (key) {
|
2013-02-11 13:50:19 +08:00
|
|
|
int offset = (tsecr) ? 3 : 0;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
|
|
|
rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
|
|
|
|
(TCPOPT_NOP << 16) |
|
|
|
|
(TCPOPT_MD5SIG << 8) |
|
|
|
|
TCPOLEN_MD5SIG);
|
|
|
|
arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
|
|
|
|
rep.th.doff = arg.iov[0].iov_len/4;
|
|
|
|
|
2008-07-19 15:01:42 +08:00
|
|
|
tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
|
2008-08-01 11:49:48 +08:00
|
|
|
key, ip_hdr(skb)->saddr,
|
|
|
|
ip_hdr(skb)->daddr, &rep.th);
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
#endif
|
2008-10-01 22:41:00 +08:00
|
|
|
arg.flags = reply_flags;
|
2007-04-21 13:47:35 +08:00
|
|
|
arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
|
|
|
|
ip_hdr(skb)->saddr, /* XXX */
|
2005-04-17 06:20:36 +08:00
|
|
|
arg.iov[0].iov_len, IPPROTO_TCP, 0);
|
|
|
|
arg.csumoffset = offsetof(struct tcphdr, check) / 2;
|
2008-04-18 11:45:16 +08:00
|
|
|
if (oif)
|
|
|
|
arg.bound_dev_if = oif;
|
2011-10-24 15:06:21 +08:00
|
|
|
arg.tos = tos;
|
2016-11-04 01:23:43 +08:00
|
|
|
arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
|
2016-05-07 00:46:18 +08:00
|
|
|
local_bh_disable();
|
2019-06-01 10:17:33 +08:00
|
|
|
ctl_sk = this_cpu_read(*net->ipv4.tcp_sk);
|
tcp: add optional per socket transmit delay
Adding delays to TCP flows is crucial for studying behavior
of TCP stacks, including congestion control modules.
Linux offers netem module, but it has unpractical constraints :
- Need root access to change qdisc
- Hard to setup on egress if combined with non trivial qdisc like FQ
- Single delay for all flows.
EDT (Earliest Departure Time) adoption in TCP stack allows us
to enable a per socket delay at a very small cost.
Networking tools can now establish thousands of flows, each of them
with a different delay, simulating real world conditions.
This requires FQ packet scheduler or a EDT-enabled NIC.
This patchs adds TCP_TX_DELAY socket option, to set a delay in
usec units.
unsigned int tx_delay = 10000; /* 10 msec */
setsockopt(fd, SOL_TCP, TCP_TX_DELAY, &tx_delay, sizeof(tx_delay));
Note that FQ packet scheduler limits might need some tweaking :
man tc-fq
PARAMETERS
limit
Hard limit on the real queue size. When this limit is
reached, new packets are dropped. If the value is lowered,
packets are dropped so that the new limit is met. Default
is 10000 packets.
flow_limit
Hard limit on the maximum number of packets queued per
flow. Default value is 100.
Use of TCP_TX_DELAY option will increase number of skbs in FQ qdisc,
so packets would be dropped if any of the previous limit is hit.
Use of a jump label makes this support runtime-free, for hosts
never using the option.
Also note that TSQ (TCP Small Queues) limits are slightly changed
with this patch : we need to account that skbs artificially delayed
wont stop us providind more skbs to feed the pipe (netem uses
skb_orphan_partial() for this purpose, but FQ can not use this trick)
Because of that, using big delays might very well trigger
old bugs in TSO auto defer logic and/or sndbuf limited detection.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-13 02:57:25 +08:00
|
|
|
ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
|
|
|
|
inet_twsk(sk)->tw_mark : sk->sk_mark;
|
2019-09-24 23:01:16 +08:00
|
|
|
ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
|
|
|
|
inet_twsk(sk)->tw_priority : sk->sk_priority;
|
2019-06-14 12:22:35 +08:00
|
|
|
transmit_time = tcp_transmit_time(sk);
|
2018-05-10 14:53:51 +08:00
|
|
|
ip_send_unicast_reply(ctl_sk,
|
2015-01-30 13:35:05 +08:00
|
|
|
skb, &TCP_SKB_CB(skb)->header.h4.opt,
|
2014-09-28 00:50:55 +08:00
|
|
|
ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
|
2019-06-14 12:22:35 +08:00
|
|
|
&arg, arg.iov[0].iov_len,
|
|
|
|
transmit_time);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-05-10 14:53:51 +08:00
|
|
|
ctl_sk->sk_mark = 0;
|
2016-04-28 07:44:32 +08:00
|
|
|
__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
|
2016-05-07 00:46:18 +08:00
|
|
|
local_bh_enable();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
2005-08-10 11:09:30 +08:00
|
|
|
struct inet_timewait_sock *tw = inet_twsk(sk);
|
2006-11-15 11:07:45 +08:00
|
|
|
struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-11-04 01:23:43 +08:00
|
|
|
tcp_v4_send_ack(sk, skb,
|
2016-01-22 00:02:54 +08:00
|
|
|
tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
|
2006-11-17 20:57:30 +08:00
|
|
|
tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
|
2017-05-17 05:00:14 +08:00
|
|
|
tcp_time_stamp_raw() + tcptw->tw_ts_offset,
|
2008-04-18 11:45:16 +08:00
|
|
|
tcptw->tw_ts_recent,
|
|
|
|
tw->tw_bound_dev_if,
|
2008-10-01 22:41:00 +08:00
|
|
|
tcp_twsk_md5_key(tcptw),
|
2011-10-24 15:06:21 +08:00
|
|
|
tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
|
|
|
|
tw->tw_tos
|
2008-04-18 11:45:16 +08:00
|
|
|
);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-08-10 11:09:30 +08:00
|
|
|
inet_twsk_put(tw);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2015-09-29 22:42:39 +08:00
|
|
|
static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
|
2006-11-17 20:57:30 +08:00
|
|
|
struct request_sock *req)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2019-12-31 06:14:25 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2019-12-31 06:14:28 +08:00
|
|
|
int l3index;
|
2019-12-31 06:14:25 +08:00
|
|
|
|
2012-08-31 20:29:13 +08:00
|
|
|
/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
|
|
|
|
* sk->sk_state == TCP_SYN_RECV -> for Fast Open.
|
|
|
|
*/
|
2016-01-22 00:02:54 +08:00
|
|
|
u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
|
|
|
|
tcp_sk(sk)->snd_nxt;
|
|
|
|
|
tcp: properly scale window in tcp_v[46]_reqsk_send_ack()
When sending an ack in SYN_RECV state, we must scale the offered
window if wscale option was negotiated and accepted.
Tested:
Following packetdrill test demonstrates the issue :
0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
+0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
+0 bind(3, ..., ...) = 0
+0 listen(3, 1) = 0
// Establish a connection.
+0 < S 0:0(0) win 20000 <mss 1000,sackOK,wscale 7, nop, TS val 100 ecr 0>
+0 > S. 0:0(0) ack 1 win 28960 <mss 1460,sackOK, TS val 100 ecr 100, nop, wscale 7>
+0 < . 1:11(10) ack 1 win 156 <nop,nop,TS val 99 ecr 100>
// check that window is properly scaled !
+0 > . 1:1(0) ack 1 win 226 <nop,nop,TS val 200 ecr 100>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Cc: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-08-23 02:31:10 +08:00
|
|
|
/* RFC 7323 2.3
|
|
|
|
* The window field (SEG.WND) of every outgoing segment, with the
|
|
|
|
* exception of <SYN> segments, MUST be right-shifted by
|
|
|
|
* Rcv.Wind.Shift bits:
|
|
|
|
*/
|
2019-12-31 06:14:25 +08:00
|
|
|
addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
2019-12-31 06:14:28 +08:00
|
|
|
l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
|
2016-11-04 01:23:43 +08:00
|
|
|
tcp_v4_send_ack(sk, skb, seq,
|
tcp: properly scale window in tcp_v[46]_reqsk_send_ack()
When sending an ack in SYN_RECV state, we must scale the offered
window if wscale option was negotiated and accepted.
Tested:
Following packetdrill test demonstrates the issue :
0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
+0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
+0 bind(3, ..., ...) = 0
+0 listen(3, 1) = 0
// Establish a connection.
+0 < S 0:0(0) win 20000 <mss 1000,sackOK,wscale 7, nop, TS val 100 ecr 0>
+0 > S. 0:0(0) ack 1 win 28960 <mss 1460,sackOK, TS val 100 ecr 100, nop, wscale 7>
+0 < . 1:11(10) ack 1 win 156 <nop,nop,TS val 99 ecr 100>
// check that window is properly scaled !
+0 > . 1:1(0) ack 1 win 226 <nop,nop,TS val 200 ecr 100>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Cc: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-08-23 02:31:10 +08:00
|
|
|
tcp_rsk(req)->rcv_nxt,
|
|
|
|
req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
|
2017-05-17 05:00:14 +08:00
|
|
|
tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
|
2008-04-18 11:45:16 +08:00
|
|
|
req->ts_recent,
|
|
|
|
0,
|
2019-12-31 06:14:28 +08:00
|
|
|
tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
|
2011-10-24 15:06:21 +08:00
|
|
|
inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
|
|
|
|
ip_hdr(skb)->tos);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2008-02-18 14:29:19 +08:00
|
|
|
* Send a SYN-ACK after having received a SYN.
|
2005-06-19 13:47:21 +08:00
|
|
|
* This still operates on a request_sock only, not on a big
|
2005-04-17 06:20:36 +08:00
|
|
|
* socket.
|
|
|
|
*/
|
2015-09-25 22:39:21 +08:00
|
|
|
static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
|
2014-06-25 22:09:58 +08:00
|
|
|
struct flowi *fl,
|
2010-01-18 11:09:39 +08:00
|
|
|
struct request_sock *req,
|
2015-10-03 02:43:35 +08:00
|
|
|
struct tcp_fastopen_cookie *foc,
|
2020-08-21 03:00:52 +08:00
|
|
|
enum tcp_synack_type synack_type,
|
|
|
|
struct sk_buff *syn_skb)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
[NET] Generalise TCP's struct open_request minisock infrastructure
Kept this first changeset minimal, without changing existing names to
ease peer review.
Basicaly tcp_openreq_alloc now receives the or_calltable, that in turn
has two new members:
->slab, that replaces tcp_openreq_cachep
->obj_size, to inform the size of the openreq descendant for
a specific protocol
The protocol specific fields in struct open_request were moved to a
class hierarchy, with the things that are common to all connection
oriented PF_INET protocols in struct inet_request_sock, the TCP ones
in tcp_request_sock, that is an inet_request_sock, that is an
open_request.
I.e. this uses the same approach used for the struct sock class
hierarchy, with sk_prot indicating if the protocol wants to use the
open_request infrastructure by filling in sk_prot->rsk_prot with an
or_calltable.
Results? Performance is improved and TCP v4 now uses only 64 bytes per
open request minisock, down from 96 without this patch :-)
Next changeset will rename some of the structs, fields and functions
mentioned above, struct or_calltable is way unclear, better name it
struct request_sock_ops, s/struct open_request/struct request_sock/g,
etc.
Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2005-06-19 13:46:52 +08:00
|
|
|
const struct inet_request_sock *ireq = inet_rsk(req);
|
2011-05-19 06:32:03 +08:00
|
|
|
struct flowi4 fl4;
|
2005-04-17 06:20:36 +08:00
|
|
|
int err = -1;
|
2013-12-23 14:37:28 +08:00
|
|
|
struct sk_buff *skb;
|
2020-09-10 08:50:48 +08:00
|
|
|
u8 tos;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* First, grab a route. */
|
2012-07-18 05:02:46 +08:00
|
|
|
if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
|
2008-03-01 03:43:03 +08:00
|
|
|
return -1;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2020-08-21 03:00:52 +08:00
|
|
|
skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (skb) {
|
2013-10-10 06:21:29 +08:00
|
|
|
__tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2020-11-21 11:47:44 +08:00
|
|
|
tos = sock_net(sk)->ipv4.sysctl_tcp_reflect_tos ?
|
2020-12-09 01:55:08 +08:00
|
|
|
(tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
|
|
|
|
(inet_sk(sk)->tos & INET_ECN_MASK) :
|
2020-11-21 11:47:44 +08:00
|
|
|
inet_sk(sk)->tos;
|
|
|
|
|
|
|
|
if (!INET_ECN_is_capable(tos) &&
|
|
|
|
tcp_bpf_ca_needs_ecn((struct sock *)req))
|
|
|
|
tos |= INET_ECN_ECT_0;
|
|
|
|
|
2018-10-03 03:35:05 +08:00
|
|
|
rcu_read_lock();
|
2013-10-10 06:21:29 +08:00
|
|
|
err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
|
|
|
|
ireq->ir_rmt_addr,
|
2020-09-10 08:50:47 +08:00
|
|
|
rcu_dereference(ireq->ireq_opt),
|
2020-11-20 05:23:51 +08:00
|
|
|
tos);
|
2018-10-03 03:35:05 +08:00
|
|
|
rcu_read_unlock();
|
2006-11-14 21:21:36 +08:00
|
|
|
err = net_xmit_eval(err);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2005-06-19 13:47:21 +08:00
|
|
|
* IPv4 request_sock destructor.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2005-06-19 13:47:21 +08:00
|
|
|
static void tcp_v4_reqsk_destructor(struct request_sock *req)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2017-10-21 00:04:13 +08:00
|
|
|
kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
|
|
|
/*
|
|
|
|
* RFC2385 MD5 checksumming requires a mapping of
|
|
|
|
* IP address->MD5 Key.
|
|
|
|
* We need to maintain these in the sk structure.
|
|
|
|
*/
|
|
|
|
|
2019-02-27 01:49:11 +08:00
|
|
|
DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
|
2018-11-28 07:03:21 +08:00
|
|
|
EXPORT_SYMBOL(tcp_md5_needed);
|
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
/* Find the Key structure for an address. */
|
2019-12-31 06:14:28 +08:00
|
|
|
struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
|
2018-11-28 07:03:21 +08:00
|
|
|
const union tcp_md5_addr *addr,
|
|
|
|
int family)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
2015-03-25 06:58:56 +08:00
|
|
|
const struct tcp_sock *tp = tcp_sk(sk);
|
2012-01-31 13:18:33 +08:00
|
|
|
struct tcp_md5sig_key *key;
|
2015-03-25 06:58:56 +08:00
|
|
|
const struct tcp_md5sig_info *md5sig;
|
2017-06-16 09:07:06 +08:00
|
|
|
__be32 mask;
|
|
|
|
struct tcp_md5sig_key *best_match = NULL;
|
|
|
|
bool match;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2012-02-01 02:45:40 +08:00
|
|
|
/* caller either holds rcu_read_lock() or socket lock */
|
|
|
|
md5sig = rcu_dereference_check(tp->md5sig_info,
|
2016-04-05 23:10:15 +08:00
|
|
|
lockdep_sock_is_held(sk));
|
2012-02-01 02:45:40 +08:00
|
|
|
if (!md5sig)
|
2006-11-15 11:07:45 +08:00
|
|
|
return NULL;
|
2017-06-21 04:11:21 +08:00
|
|
|
|
2020-02-22 01:57:14 +08:00
|
|
|
hlist_for_each_entry_rcu(key, &md5sig->head, node,
|
|
|
|
lockdep_sock_is_held(sk)) {
|
2012-01-31 13:18:33 +08:00
|
|
|
if (key->family != family)
|
|
|
|
continue;
|
2019-12-31 06:14:28 +08:00
|
|
|
if (key->l3index && key->l3index != l3index)
|
|
|
|
continue;
|
2017-06-16 09:07:06 +08:00
|
|
|
if (family == AF_INET) {
|
|
|
|
mask = inet_make_mask(key->prefixlen);
|
|
|
|
match = (key->addr.a4.s_addr & mask) ==
|
|
|
|
(addr->a4.s_addr & mask);
|
|
|
|
#if IS_ENABLED(CONFIG_IPV6)
|
|
|
|
} else if (family == AF_INET6) {
|
|
|
|
match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
|
|
|
|
key->prefixlen);
|
|
|
|
#endif
|
|
|
|
} else {
|
|
|
|
match = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (match && (!best_match ||
|
|
|
|
key->prefixlen > best_match->prefixlen))
|
|
|
|
best_match = key;
|
|
|
|
}
|
|
|
|
return best_match;
|
|
|
|
}
|
2018-11-28 07:03:21 +08:00
|
|
|
EXPORT_SYMBOL(__tcp_md5_do_lookup);
|
2017-06-16 09:07:06 +08:00
|
|
|
|
2017-07-06 07:58:53 +08:00
|
|
|
static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
|
|
|
|
const union tcp_md5_addr *addr,
|
2019-12-31 06:14:28 +08:00
|
|
|
int family, u8 prefixlen,
|
|
|
|
int l3index)
|
2017-06-16 09:07:06 +08:00
|
|
|
{
|
|
|
|
const struct tcp_sock *tp = tcp_sk(sk);
|
|
|
|
struct tcp_md5sig_key *key;
|
|
|
|
unsigned int size = sizeof(struct in_addr);
|
|
|
|
const struct tcp_md5sig_info *md5sig;
|
|
|
|
|
|
|
|
/* caller either holds rcu_read_lock() or socket lock */
|
|
|
|
md5sig = rcu_dereference_check(tp->md5sig_info,
|
|
|
|
lockdep_sock_is_held(sk));
|
|
|
|
if (!md5sig)
|
|
|
|
return NULL;
|
|
|
|
#if IS_ENABLED(CONFIG_IPV6)
|
|
|
|
if (family == AF_INET6)
|
|
|
|
size = sizeof(struct in6_addr);
|
|
|
|
#endif
|
2020-02-22 01:57:14 +08:00
|
|
|
hlist_for_each_entry_rcu(key, &md5sig->head, node,
|
|
|
|
lockdep_sock_is_held(sk)) {
|
2017-06-16 09:07:06 +08:00
|
|
|
if (key->family != family)
|
|
|
|
continue;
|
2019-12-31 06:14:28 +08:00
|
|
|
if (key->l3index && key->l3index != l3index)
|
|
|
|
continue;
|
2017-06-16 09:07:06 +08:00
|
|
|
if (!memcmp(&key->addr, addr, size) &&
|
|
|
|
key->prefixlen == prefixlen)
|
2012-01-31 13:18:33 +08:00
|
|
|
return key;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2015-09-25 22:39:15 +08:00
|
|
|
struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
|
2015-03-25 06:58:56 +08:00
|
|
|
const struct sock *addr_sk)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
2015-04-10 05:36:42 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2019-12-31 06:14:28 +08:00
|
|
|
int l3index;
|
2012-01-31 13:18:33 +08:00
|
|
|
|
2019-12-31 06:14:28 +08:00
|
|
|
l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
|
|
|
|
addr_sk->sk_bound_dev_if);
|
2015-04-10 05:36:42 +08:00
|
|
|
addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
|
2019-12-31 06:14:28 +08:00
|
|
|
return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(tcp_v4_md5_lookup);
|
|
|
|
|
|
|
|
/* This can be called on a newly created socket, from other files */
|
2012-01-31 13:18:33 +08:00
|
|
|
int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
|
2019-12-31 06:14:28 +08:00
|
|
|
int family, u8 prefixlen, int l3index,
|
|
|
|
const u8 *newkey, u8 newkeylen, gfp_t gfp)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
|
|
|
/* Add Key to the list */
|
2007-10-30 11:55:27 +08:00
|
|
|
struct tcp_md5sig_key *key;
|
2006-11-15 11:07:45 +08:00
|
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
2012-01-31 13:18:33 +08:00
|
|
|
struct tcp_md5sig_info *md5sig;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2019-12-31 06:14:28 +08:00
|
|
|
key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
|
2006-11-15 11:07:45 +08:00
|
|
|
if (key) {
|
2020-07-02 02:43:04 +08:00
|
|
|
/* Pre-existing entry - just update that one.
|
|
|
|
* Note that the key might be used concurrently.
|
|
|
|
* data_race() is telling kcsan that we do not care of
|
|
|
|
* key mismatches, since changing MD5 key on live flows
|
|
|
|
* can lead to packet drops.
|
|
|
|
*/
|
|
|
|
data_race(memcpy(key->key, newkey, newkeylen));
|
2020-07-01 07:41:01 +08:00
|
|
|
|
2020-07-02 02:43:04 +08:00
|
|
|
/* Pairs with READ_ONCE() in tcp_md5_hash_key().
|
|
|
|
* Also note that a reader could catch new key->keylen value
|
|
|
|
* but old key->key[], this is the reason we use __GFP_ZERO
|
|
|
|
* at sock_kmalloc() time below these lines.
|
|
|
|
*/
|
|
|
|
WRITE_ONCE(key->keylen, newkeylen);
|
2020-07-01 07:41:01 +08:00
|
|
|
|
2012-01-31 13:18:33 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2011-09-30 01:10:10 +08:00
|
|
|
|
2012-02-01 02:45:40 +08:00
|
|
|
md5sig = rcu_dereference_protected(tp->md5sig_info,
|
2016-04-05 23:10:15 +08:00
|
|
|
lockdep_sock_is_held(sk));
|
2012-01-31 13:18:33 +08:00
|
|
|
if (!md5sig) {
|
|
|
|
md5sig = kmalloc(sizeof(*md5sig), gfp);
|
|
|
|
if (!md5sig)
|
2006-11-15 11:07:45 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2012-01-31 13:18:33 +08:00
|
|
|
sk_nocaps_add(sk, NETIF_F_GSO_MASK);
|
|
|
|
INIT_HLIST_HEAD(&md5sig->head);
|
2012-02-01 02:45:40 +08:00
|
|
|
rcu_assign_pointer(tp->md5sig_info, md5sig);
|
2012-01-31 13:18:33 +08:00
|
|
|
}
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2020-07-02 02:43:04 +08:00
|
|
|
key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
|
2012-01-31 13:18:33 +08:00
|
|
|
if (!key)
|
|
|
|
return -ENOMEM;
|
2013-05-20 14:52:26 +08:00
|
|
|
if (!tcp_alloc_md5sig_pool()) {
|
2012-01-31 18:56:48 +08:00
|
|
|
sock_kfree_s(sk, key, sizeof(*key));
|
2012-01-31 13:18:33 +08:00
|
|
|
return -ENOMEM;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2012-01-31 13:18:33 +08:00
|
|
|
|
|
|
|
memcpy(key->key, newkey, newkeylen);
|
|
|
|
key->keylen = newkeylen;
|
|
|
|
key->family = family;
|
2017-06-16 09:07:06 +08:00
|
|
|
key->prefixlen = prefixlen;
|
2019-12-31 06:14:28 +08:00
|
|
|
key->l3index = l3index;
|
2012-01-31 13:18:33 +08:00
|
|
|
memcpy(&key->addr, addr,
|
|
|
|
(family == AF_INET6) ? sizeof(struct in6_addr) :
|
|
|
|
sizeof(struct in_addr));
|
|
|
|
hlist_add_head_rcu(&key->node, &md5sig->head);
|
2006-11-15 11:07:45 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2012-01-31 13:18:33 +08:00
|
|
|
EXPORT_SYMBOL(tcp_md5_do_add);
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2017-06-16 09:07:06 +08:00
|
|
|
int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
|
2019-12-31 06:14:28 +08:00
|
|
|
u8 prefixlen, int l3index)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
2012-01-31 13:18:33 +08:00
|
|
|
struct tcp_md5sig_key *key;
|
|
|
|
|
2019-12-31 06:14:28 +08:00
|
|
|
key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index);
|
2012-01-31 13:18:33 +08:00
|
|
|
if (!key)
|
|
|
|
return -ENOENT;
|
|
|
|
hlist_del_rcu(&key->node);
|
2012-01-31 18:56:48 +08:00
|
|
|
atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
|
2012-01-31 13:18:33 +08:00
|
|
|
kfree_rcu(key, rcu);
|
|
|
|
return 0;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2012-01-31 13:18:33 +08:00
|
|
|
EXPORT_SYMBOL(tcp_md5_do_del);
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2012-10-26 22:31:40 +08:00
|
|
|
static void tcp_clear_md5_list(struct sock *sk)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
|
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
2012-01-31 13:18:33 +08:00
|
|
|
struct tcp_md5sig_key *key;
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 09:06:00 +08:00
|
|
|
struct hlist_node *n;
|
2012-02-01 02:45:40 +08:00
|
|
|
struct tcp_md5sig_info *md5sig;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2012-02-01 02:45:40 +08:00
|
|
|
md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
|
|
|
|
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 09:06:00 +08:00
|
|
|
hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
|
2012-01-31 13:18:33 +08:00
|
|
|
hlist_del_rcu(&key->node);
|
2012-01-31 18:56:48 +08:00
|
|
|
atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
|
2012-01-31 13:18:33 +08:00
|
|
|
kfree_rcu(key, rcu);
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-16 09:07:07 +08:00
|
|
|
static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
|
2020-07-23 14:09:05 +08:00
|
|
|
sockptr_t optval, int optlen)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
|
|
|
struct tcp_md5sig cmd;
|
|
|
|
struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
|
2019-12-31 06:14:25 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2017-06-16 09:07:07 +08:00
|
|
|
u8 prefixlen = 32;
|
2019-12-31 06:14:28 +08:00
|
|
|
int l3index = 0;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
|
|
|
if (optlen < sizeof(cmd))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2020-07-23 14:09:05 +08:00
|
|
|
if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
|
2006-11-15 11:07:45 +08:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
if (sin->sin_family != AF_INET)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2017-06-16 09:07:07 +08:00
|
|
|
if (optname == TCP_MD5SIG_EXT &&
|
|
|
|
cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
|
|
|
|
prefixlen = cmd.tcpm_prefixlen;
|
|
|
|
if (prefixlen > 32)
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2019-12-31 06:14:29 +08:00
|
|
|
if (optname == TCP_MD5SIG_EXT &&
|
|
|
|
cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
|
|
|
|
struct net_device *dev;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
|
|
|
|
if (dev && netif_is_l3_master(dev))
|
|
|
|
l3index = dev->ifindex;
|
|
|
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
/* ok to reference set/not set outside of rcu;
|
|
|
|
* right now device MUST be an L3 master
|
|
|
|
*/
|
|
|
|
if (!dev || !l3index)
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2019-12-31 06:14:25 +08:00
|
|
|
addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
|
|
|
|
|
2014-08-04 02:45:19 +08:00
|
|
|
if (!cmd.tcpm_keylen)
|
2019-12-31 06:14:28 +08:00
|
|
|
return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index);
|
2006-11-15 11:07:45 +08:00
|
|
|
|
|
|
|
if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2019-12-31 06:14:28 +08:00
|
|
|
return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index,
|
2019-12-31 06:14:25 +08:00
|
|
|
cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
|
2016-06-28 00:51:53 +08:00
|
|
|
static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
|
|
|
|
__be32 daddr, __be32 saddr,
|
|
|
|
const struct tcphdr *th, int nbytes)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
|
|
|
struct tcp4_pseudohdr *bp;
|
2008-07-19 15:01:42 +08:00
|
|
|
struct scatterlist sg;
|
2016-06-28 00:51:53 +08:00
|
|
|
struct tcphdr *_th;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2016-06-28 00:51:53 +08:00
|
|
|
bp = hp->scratch;
|
2006-11-15 11:07:45 +08:00
|
|
|
bp->saddr = saddr;
|
|
|
|
bp->daddr = daddr;
|
|
|
|
bp->pad = 0;
|
2008-04-17 11:48:12 +08:00
|
|
|
bp->protocol = IPPROTO_TCP;
|
2008-07-19 15:01:42 +08:00
|
|
|
bp->len = cpu_to_be16(nbytes);
|
2007-10-26 15:41:21 +08:00
|
|
|
|
2016-06-28 00:51:53 +08:00
|
|
|
_th = (struct tcphdr *)(bp + 1);
|
|
|
|
memcpy(_th, th, sizeof(*th));
|
|
|
|
_th->check = 0;
|
|
|
|
|
|
|
|
sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
|
|
|
|
ahash_request_set_crypt(hp->md5_req, &sg, NULL,
|
|
|
|
sizeof(*bp) + sizeof(*th));
|
2016-01-24 21:20:23 +08:00
|
|
|
return crypto_ahash_update(hp->md5_req);
|
2008-07-19 15:01:42 +08:00
|
|
|
}
|
|
|
|
|
2012-01-31 13:18:33 +08:00
|
|
|
static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
|
2011-10-24 14:46:04 +08:00
|
|
|
__be32 daddr, __be32 saddr, const struct tcphdr *th)
|
2008-07-19 15:01:42 +08:00
|
|
|
{
|
|
|
|
struct tcp_md5sig_pool *hp;
|
2016-01-24 21:20:23 +08:00
|
|
|
struct ahash_request *req;
|
2008-07-19 15:01:42 +08:00
|
|
|
|
|
|
|
hp = tcp_get_md5sig_pool();
|
|
|
|
if (!hp)
|
|
|
|
goto clear_hash_noput;
|
2016-01-24 21:20:23 +08:00
|
|
|
req = hp->md5_req;
|
2008-07-19 15:01:42 +08:00
|
|
|
|
2016-01-24 21:20:23 +08:00
|
|
|
if (crypto_ahash_init(req))
|
2008-07-19 15:01:42 +08:00
|
|
|
goto clear_hash;
|
2016-06-28 00:51:53 +08:00
|
|
|
if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
|
2008-07-19 15:01:42 +08:00
|
|
|
goto clear_hash;
|
|
|
|
if (tcp_md5_hash_key(hp, key))
|
|
|
|
goto clear_hash;
|
2016-01-24 21:20:23 +08:00
|
|
|
ahash_request_set_crypt(req, NULL, md5_hash, 0);
|
|
|
|
if (crypto_ahash_final(req))
|
2006-11-15 11:07:45 +08:00
|
|
|
goto clear_hash;
|
|
|
|
|
|
|
|
tcp_put_md5sig_pool();
|
|
|
|
return 0;
|
2008-07-19 15:01:42 +08:00
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
clear_hash:
|
|
|
|
tcp_put_md5sig_pool();
|
|
|
|
clear_hash_noput:
|
|
|
|
memset(md5_hash, 0, 16);
|
2008-07-19 15:01:42 +08:00
|
|
|
return 1;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
|
2015-03-25 06:58:55 +08:00
|
|
|
int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
|
|
|
|
const struct sock *sk,
|
2011-10-24 14:46:04 +08:00
|
|
|
const struct sk_buff *skb)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
2008-07-19 15:01:42 +08:00
|
|
|
struct tcp_md5sig_pool *hp;
|
2016-01-24 21:20:23 +08:00
|
|
|
struct ahash_request *req;
|
2011-10-24 14:46:04 +08:00
|
|
|
const struct tcphdr *th = tcp_hdr(skb);
|
2006-11-15 11:07:45 +08:00
|
|
|
__be32 saddr, daddr;
|
|
|
|
|
2015-03-25 06:58:55 +08:00
|
|
|
if (sk) { /* valid for establish/request sockets */
|
|
|
|
saddr = sk->sk_rcv_saddr;
|
|
|
|
daddr = sk->sk_daddr;
|
2006-11-15 11:07:45 +08:00
|
|
|
} else {
|
2008-07-19 15:01:42 +08:00
|
|
|
const struct iphdr *iph = ip_hdr(skb);
|
|
|
|
saddr = iph->saddr;
|
|
|
|
daddr = iph->daddr;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2008-07-19 15:01:42 +08:00
|
|
|
|
|
|
|
hp = tcp_get_md5sig_pool();
|
|
|
|
if (!hp)
|
|
|
|
goto clear_hash_noput;
|
2016-01-24 21:20:23 +08:00
|
|
|
req = hp->md5_req;
|
2008-07-19 15:01:42 +08:00
|
|
|
|
2016-01-24 21:20:23 +08:00
|
|
|
if (crypto_ahash_init(req))
|
2008-07-19 15:01:42 +08:00
|
|
|
goto clear_hash;
|
|
|
|
|
2016-06-28 00:51:53 +08:00
|
|
|
if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
|
2008-07-19 15:01:42 +08:00
|
|
|
goto clear_hash;
|
|
|
|
if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
|
|
|
|
goto clear_hash;
|
|
|
|
if (tcp_md5_hash_key(hp, key))
|
|
|
|
goto clear_hash;
|
2016-01-24 21:20:23 +08:00
|
|
|
ahash_request_set_crypt(req, NULL, md5_hash, 0);
|
|
|
|
if (crypto_ahash_final(req))
|
2008-07-19 15:01:42 +08:00
|
|
|
goto clear_hash;
|
|
|
|
|
|
|
|
tcp_put_md5sig_pool();
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
clear_hash:
|
|
|
|
tcp_put_md5sig_pool();
|
|
|
|
clear_hash_noput:
|
|
|
|
memset(md5_hash, 0, 16);
|
|
|
|
return 1;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2008-07-19 15:01:42 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2015-10-03 02:43:28 +08:00
|
|
|
#endif
|
|
|
|
|
2015-03-25 06:58:54 +08:00
|
|
|
/* Called with rcu_read_lock() */
|
2015-10-03 02:43:28 +08:00
|
|
|
static bool tcp_v4_inbound_md5_hash(const struct sock *sk,
|
2019-12-31 06:14:27 +08:00
|
|
|
const struct sk_buff *skb,
|
|
|
|
int dif, int sdif)
|
2006-11-15 11:07:45 +08:00
|
|
|
{
|
2015-10-03 02:43:28 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2006-11-15 11:07:45 +08:00
|
|
|
/*
|
|
|
|
* This gets called for each TCP segment that arrives
|
|
|
|
* so we want to be efficient.
|
|
|
|
* We have 3 drop cases:
|
|
|
|
* o No MD5 hash and one expected.
|
|
|
|
* o MD5 hash and we're not expecting one.
|
|
|
|
* o MD5 hash and its wrong.
|
|
|
|
*/
|
2011-10-21 17:22:42 +08:00
|
|
|
const __u8 *hash_location = NULL;
|
2006-11-15 11:07:45 +08:00
|
|
|
struct tcp_md5sig_key *hash_expected;
|
2007-04-21 13:47:35 +08:00
|
|
|
const struct iphdr *iph = ip_hdr(skb);
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct tcphdr *th = tcp_hdr(skb);
|
2019-12-31 06:14:25 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2006-11-15 11:07:45 +08:00
|
|
|
unsigned char newhash[16];
|
2019-12-31 06:14:28 +08:00
|
|
|
int genhash, l3index;
|
|
|
|
|
|
|
|
/* sdif set, means packet ingressed via a device
|
|
|
|
* in an L3 domain and dif is set to the l3mdev
|
|
|
|
*/
|
|
|
|
l3index = sdif ? dif : 0;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2019-12-31 06:14:25 +08:00
|
|
|
addr = (union tcp_md5_addr *)&iph->saddr;
|
2019-12-31 06:14:28 +08:00
|
|
|
hash_expected = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
2008-04-17 11:29:53 +08:00
|
|
|
hash_location = tcp_parse_md5sig_option(th);
|
2006-11-15 11:07:45 +08:00
|
|
|
|
|
|
|
/* We've parsed the options - do we have a hash? */
|
|
|
|
if (!hash_expected && !hash_location)
|
2012-05-17 07:15:34 +08:00
|
|
|
return false;
|
2006-11-15 11:07:45 +08:00
|
|
|
|
|
|
|
if (hash_expected && !hash_location) {
|
2016-04-30 05:16:47 +08:00
|
|
|
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
|
2012-05-17 07:15:34 +08:00
|
|
|
return true;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!hash_expected && hash_location) {
|
2016-04-30 05:16:47 +08:00
|
|
|
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
|
2012-05-17 07:15:34 +08:00
|
|
|
return true;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Okay, so this is hash_expected and hash_location -
|
|
|
|
* so we need to calculate the checksum.
|
|
|
|
*/
|
2008-07-19 15:01:42 +08:00
|
|
|
genhash = tcp_v4_md5_hash_skb(newhash,
|
|
|
|
hash_expected,
|
2015-03-25 06:58:55 +08:00
|
|
|
NULL, skb);
|
2006-11-15 11:07:45 +08:00
|
|
|
|
|
|
|
if (genhash || memcmp(hash_location, newhash, 16) != 0) {
|
2016-08-25 00:01:23 +08:00
|
|
|
NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
|
2019-12-31 06:14:28 +08:00
|
|
|
net_info_ratelimited("MD5 Hash failed for (%pI4, %d)->(%pI4, %d)%s L3 index %d\n",
|
2012-05-14 05:56:26 +08:00
|
|
|
&iph->saddr, ntohs(th->source),
|
|
|
|
&iph->daddr, ntohs(th->dest),
|
|
|
|
genhash ? " tcp_v4_calc_md5_hash failed"
|
2019-12-31 06:14:28 +08:00
|
|
|
: "", l3index);
|
2012-05-17 07:15:34 +08:00
|
|
|
return true;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2012-05-17 07:15:34 +08:00
|
|
|
return false;
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
2015-10-03 02:43:28 +08:00
|
|
|
return false;
|
|
|
|
}
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2015-09-25 22:39:08 +08:00
|
|
|
static void tcp_v4_init_req(struct request_sock *req,
|
|
|
|
const struct sock *sk_listener,
|
2014-06-25 22:09:53 +08:00
|
|
|
struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct inet_request_sock *ireq = inet_rsk(req);
|
2017-10-21 00:04:13 +08:00
|
|
|
struct net *net = sock_net(sk_listener);
|
2014-06-25 22:09:53 +08:00
|
|
|
|
2015-03-19 05:05:38 +08:00
|
|
|
sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
|
|
|
|
sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
|
2017-10-21 00:04:13 +08:00
|
|
|
RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
|
2014-06-25 22:09:53 +08:00
|
|
|
}
|
|
|
|
|
2015-09-29 22:42:50 +08:00
|
|
|
static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
|
2020-11-30 23:36:30 +08:00
|
|
|
struct sk_buff *skb,
|
2015-09-29 22:42:50 +08:00
|
|
|
struct flowi *fl,
|
2020-11-30 23:36:30 +08:00
|
|
|
struct request_sock *req)
|
2014-06-25 22:09:55 +08:00
|
|
|
{
|
2020-11-30 23:36:30 +08:00
|
|
|
tcp_v4_init_req(req, sk, skb);
|
|
|
|
|
|
|
|
if (security_inet_conn_request(sk, skb, req))
|
|
|
|
return NULL;
|
|
|
|
|
2017-03-16 04:30:46 +08:00
|
|
|
return inet_csk_route_req(sk, &fl->u.ip4, req);
|
2014-06-25 22:09:55 +08:00
|
|
|
}
|
|
|
|
|
2006-11-16 18:30:37 +08:00
|
|
|
struct request_sock_ops tcp_request_sock_ops __read_mostly = {
|
2005-04-17 06:20:36 +08:00
|
|
|
.family = PF_INET,
|
[NET] Generalise TCP's struct open_request minisock infrastructure
Kept this first changeset minimal, without changing existing names to
ease peer review.
Basicaly tcp_openreq_alloc now receives the or_calltable, that in turn
has two new members:
->slab, that replaces tcp_openreq_cachep
->obj_size, to inform the size of the openreq descendant for
a specific protocol
The protocol specific fields in struct open_request were moved to a
class hierarchy, with the things that are common to all connection
oriented PF_INET protocols in struct inet_request_sock, the TCP ones
in tcp_request_sock, that is an inet_request_sock, that is an
open_request.
I.e. this uses the same approach used for the struct sock class
hierarchy, with sk_prot indicating if the protocol wants to use the
open_request infrastructure by filling in sk_prot->rsk_prot with an
or_calltable.
Results? Performance is improved and TCP v4 now uses only 64 bytes per
open request minisock, down from 96 without this patch :-)
Next changeset will rename some of the structs, fields and functions
mentioned above, struct or_calltable is way unclear, better name it
struct request_sock_ops, s/struct open_request/struct request_sock/g,
etc.
Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2005-06-19 13:46:52 +08:00
|
|
|
.obj_size = sizeof(struct tcp_request_sock),
|
2014-06-25 22:09:59 +08:00
|
|
|
.rtx_syn_ack = tcp_rtx_synack,
|
2005-06-19 13:47:21 +08:00
|
|
|
.send_ack = tcp_v4_reqsk_send_ack,
|
|
|
|
.destructor = tcp_v4_reqsk_destructor,
|
2005-04-17 06:20:36 +08:00
|
|
|
.send_reset = tcp_v4_send_reset,
|
2014-08-30 14:32:05 +08:00
|
|
|
.syn_ack_timeout = tcp_syn_ack_timeout,
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
2020-01-09 23:59:21 +08:00
|
|
|
const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
|
2014-06-25 22:10:00 +08:00
|
|
|
.mss_clamp = TCP_MSS_DEFAULT,
|
2014-06-25 22:09:53 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2015-03-25 06:58:56 +08:00
|
|
|
.req_md5_lookup = tcp_v4_md5_lookup,
|
2009-07-16 13:04:51 +08:00
|
|
|
.calc_md5_hash = tcp_v4_md5_hash_skb,
|
2006-12-01 11:16:28 +08:00
|
|
|
#endif
|
2014-06-25 22:09:54 +08:00
|
|
|
#ifdef CONFIG_SYN_COOKIES
|
|
|
|
.cookie_init_seq = cookie_v4_init_sequence,
|
|
|
|
#endif
|
2014-06-25 22:09:55 +08:00
|
|
|
.route_req = tcp_v4_route_req,
|
2017-05-05 21:56:54 +08:00
|
|
|
.init_seq = tcp_v4_init_seq,
|
|
|
|
.init_ts_off = tcp_v4_init_ts_off,
|
2014-06-25 22:09:58 +08:00
|
|
|
.send_synack = tcp_v4_send_synack,
|
2014-06-25 22:09:53 +08:00
|
|
|
};
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
/* Never answer to SYNs send to broadcast or multicast */
|
2009-06-02 13:14:27 +08:00
|
|
|
if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto drop;
|
|
|
|
|
2014-06-25 22:10:02 +08:00
|
|
|
return tcp_conn_request(&tcp_request_sock_ops,
|
|
|
|
&tcp_request_sock_ipv4_ops, sk, skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
drop:
|
2016-04-01 23:52:20 +08:00
|
|
|
tcp_listendrop(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_conn_request);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The three way handshake has completed - we got a valid synack -
|
|
|
|
* now create the new socket.
|
|
|
|
*/
|
2015-09-29 22:42:48 +08:00
|
|
|
struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
|
2005-06-19 13:47:21 +08:00
|
|
|
struct request_sock *req,
|
2015-10-22 23:20:46 +08:00
|
|
|
struct dst_entry *dst,
|
|
|
|
struct request_sock *req_unhash,
|
|
|
|
bool *own_req)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
[NET] Generalise TCP's struct open_request minisock infrastructure
Kept this first changeset minimal, without changing existing names to
ease peer review.
Basicaly tcp_openreq_alloc now receives the or_calltable, that in turn
has two new members:
->slab, that replaces tcp_openreq_cachep
->obj_size, to inform the size of the openreq descendant for
a specific protocol
The protocol specific fields in struct open_request were moved to a
class hierarchy, with the things that are common to all connection
oriented PF_INET protocols in struct inet_request_sock, the TCP ones
in tcp_request_sock, that is an inet_request_sock, that is an
open_request.
I.e. this uses the same approach used for the struct sock class
hierarchy, with sk_prot indicating if the protocol wants to use the
open_request infrastructure by filling in sk_prot->rsk_prot with an
or_calltable.
Results? Performance is improved and TCP v4 now uses only 64 bytes per
open request minisock, down from 96 without this patch :-)
Next changeset will rename some of the structs, fields and functions
mentioned above, struct or_calltable is way unclear, better name it
struct request_sock_ops, s/struct open_request/struct request_sock/g,
etc.
Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2005-06-19 13:46:52 +08:00
|
|
|
struct inet_request_sock *ireq;
|
tcp: fix race condition when creating child sockets from syncookies
When the TCP stack is in SYN flood mode, the server child socket is
created from the SYN cookie received in a TCP packet with the ACK flag
set.
The child socket is created when the server receives the first TCP
packet with a valid SYN cookie from the client. Usually, this packet
corresponds to the final step of the TCP 3-way handshake, the ACK
packet. But is also possible to receive a valid SYN cookie from the
first TCP data packet sent by the client, and thus create a child socket
from that SYN cookie.
Since a client socket is ready to send data as soon as it receives the
SYN+ACK packet from the server, the client can send the ACK packet (sent
by the TCP stack code), and the first data packet (sent by the userspace
program) almost at the same time, and thus the server will equally
receive the two TCP packets with valid SYN cookies almost at the same
instant.
When such event happens, the TCP stack code has a race condition that
occurs between the momement a lookup is done to the established
connections hashtable to check for the existence of a connection for the
same client, and the moment that the child socket is added to the
established connections hashtable. As a consequence, this race condition
can lead to a situation where we add two child sockets to the
established connections hashtable and deliver two sockets to the
userspace program to the same client.
This patch fixes the race condition by checking if an existing child
socket exists for the same client when we are adding the second child
socket to the established connections socket. If an existing child
socket exists, we drop the packet and discard the second child socket
to the same client.
Signed-off-by: Ricardo Dias <rdias@singlestore.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20201120111133.GA67501@rdias-suse-pc.lan
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2020-11-20 19:11:33 +08:00
|
|
|
bool found_dup_sk = false;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct inet_sock *newinet;
|
|
|
|
struct tcp_sock *newtp;
|
|
|
|
struct sock *newsk;
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2019-12-31 06:14:25 +08:00
|
|
|
const union tcp_md5_addr *addr;
|
2006-11-15 11:07:45 +08:00
|
|
|
struct tcp_md5sig_key *key;
|
2019-12-31 06:14:28 +08:00
|
|
|
int l3index;
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
2011-04-21 17:45:37 +08:00
|
|
|
struct ip_options_rcu *inet_opt;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (sk_acceptq_is_full(sk))
|
|
|
|
goto exit_overflow;
|
|
|
|
|
|
|
|
newsk = tcp_create_openreq_child(sk, req, skb);
|
|
|
|
if (!newsk)
|
2010-10-21 19:06:43 +08:00
|
|
|
goto exit_nonewsk;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-07-01 04:36:35 +08:00
|
|
|
newsk->sk_gso_type = SKB_GSO_TCPV4;
|
2012-08-19 11:30:38 +08:00
|
|
|
inet_sk_rx_dst_set(newsk, skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
newtp = tcp_sk(newsk);
|
|
|
|
newinet = inet_sk(newsk);
|
[NET] Generalise TCP's struct open_request minisock infrastructure
Kept this first changeset minimal, without changing existing names to
ease peer review.
Basicaly tcp_openreq_alloc now receives the or_calltable, that in turn
has two new members:
->slab, that replaces tcp_openreq_cachep
->obj_size, to inform the size of the openreq descendant for
a specific protocol
The protocol specific fields in struct open_request were moved to a
class hierarchy, with the things that are common to all connection
oriented PF_INET protocols in struct inet_request_sock, the TCP ones
in tcp_request_sock, that is an inet_request_sock, that is an
open_request.
I.e. this uses the same approach used for the struct sock class
hierarchy, with sk_prot indicating if the protocol wants to use the
open_request infrastructure by filling in sk_prot->rsk_prot with an
or_calltable.
Results? Performance is improved and TCP v4 now uses only 64 bytes per
open request minisock, down from 96 without this patch :-)
Next changeset will rename some of the structs, fields and functions
mentioned above, struct or_calltable is way unclear, better name it
struct request_sock_ops, s/struct open_request/struct request_sock/g,
etc.
Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2005-06-19 13:46:52 +08:00
|
|
|
ireq = inet_rsk(req);
|
2015-03-19 05:05:35 +08:00
|
|
|
sk_daddr_set(newsk, ireq->ir_rmt_addr);
|
|
|
|
sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
|
2015-12-17 05:20:44 +08:00
|
|
|
newsk->sk_bound_dev_if = ireq->ir_iif;
|
2017-10-21 00:04:13 +08:00
|
|
|
newinet->inet_saddr = ireq->ir_loc_addr;
|
|
|
|
inet_opt = rcu_dereference(ireq->ireq_opt);
|
|
|
|
RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
|
2005-08-10 11:10:42 +08:00
|
|
|
newinet->mc_index = inet_iif(skb);
|
2007-04-21 13:47:35 +08:00
|
|
|
newinet->mc_ttl = ip_hdr(skb)->ttl;
|
2012-02-09 17:35:49 +08:00
|
|
|
newinet->rcv_tos = ip_hdr(skb)->tos;
|
2005-12-14 15:26:10 +08:00
|
|
|
inet_csk(newsk)->icsk_ext_hdr_len = 0;
|
2011-04-21 17:45:37 +08:00
|
|
|
if (inet_opt)
|
|
|
|
inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
|
2019-11-02 01:32:19 +08:00
|
|
|
newinet->inet_id = prandom_u32();
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2020-12-09 01:55:08 +08:00
|
|
|
/* Set ToS of the new socket based upon the value of incoming SYN.
|
|
|
|
* ECT bits are set later in tcp_init_transfer().
|
|
|
|
*/
|
2020-09-10 08:50:48 +08:00
|
|
|
if (sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)
|
|
|
|
newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
|
|
|
|
|
2012-03-10 17:20:21 +08:00
|
|
|
if (!dst) {
|
|
|
|
dst = inet_csk_route_child_sock(sk, newsk, req);
|
|
|
|
if (!dst)
|
|
|
|
goto put_and_exit;
|
|
|
|
} else {
|
|
|
|
/* syncookie case : see end of cookie_v4_check() */
|
|
|
|
}
|
2011-05-09 06:28:03 +08:00
|
|
|
sk_setup_caps(newsk, dst);
|
|
|
|
|
net: tcp: add per route congestion control
This work adds the possibility to define a per route/destination
congestion control algorithm. Generally, this opens up the possibility
for a machine with different links to enforce specific congestion
control algorithms with optimal strategies for each of them based
on their network characteristics, even transparently for a single
application listening on all links.
For our specific use case, this additionally facilitates deployment
of DCTCP, for example, applications can easily serve internal
traffic/dsts in DCTCP and external one with CUBIC. Other scenarios
would also allow for utilizing e.g. long living, low priority
background flows for certain destinations/routes while still being
able for normal traffic to utilize the default congestion control
algorithm. We also thought about a per netns setting (where different
defaults are possible), but given its actually a link specific
property, we argue that a per route/destination setting is the most
natural and flexible.
The administrator can utilize this through ip-route(8) by appending
"congctl [lock] <name>", where <name> denotes the name of a
congestion control algorithm and the optional lock parameter allows
to enforce the given algorithm so that applications in user space
would not be allowed to overwrite that algorithm for that destination.
The dst metric lookups are being done when a dst entry is already
available in order to avoid a costly lookup and still before the
algorithms are being initialized, thus overhead is very low when the
feature is not being used. While the client side would need to drop
the current reference on the module, on server side this can actually
even be avoided as we just got a flat-copied socket clone.
Joint work with Florian Westphal.
Suggested-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-01-06 06:57:48 +08:00
|
|
|
tcp_ca_openreq_child(newsk, dst);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
tcp_sync_mss(newsk, dst_mtu(dst));
|
2017-02-03 00:04:56 +08:00
|
|
|
newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
|
2008-09-21 15:21:51 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
tcp_initialize_rcv_mss(newsk);
|
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2019-12-31 06:14:28 +08:00
|
|
|
l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
|
2006-11-15 11:07:45 +08:00
|
|
|
/* Copy over the MD5 key from the original socket */
|
2019-12-31 06:14:25 +08:00
|
|
|
addr = (union tcp_md5_addr *)&newinet->inet_daddr;
|
2019-12-31 06:14:28 +08:00
|
|
|
key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
2015-04-03 16:17:27 +08:00
|
|
|
if (key) {
|
2006-11-15 11:07:45 +08:00
|
|
|
/*
|
|
|
|
* We're using one, so create a matching key
|
|
|
|
* on the newsk structure. If we fail to get
|
|
|
|
* memory, then we end up not copying the key
|
|
|
|
* across. Shucks.
|
|
|
|
*/
|
2019-12-31 06:14:28 +08:00
|
|
|
tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index,
|
2019-12-31 06:14:25 +08:00
|
|
|
key->key, key->keylen, GFP_ATOMIC);
|
2010-05-16 15:36:33 +08:00
|
|
|
sk_nocaps_add(newsk, NETIF_F_GSO_MASK);
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2011-05-09 06:28:03 +08:00
|
|
|
if (__inet_inherit_port(sk, newsk) < 0)
|
|
|
|
goto put_and_exit;
|
tcp: fix race condition when creating child sockets from syncookies
When the TCP stack is in SYN flood mode, the server child socket is
created from the SYN cookie received in a TCP packet with the ACK flag
set.
The child socket is created when the server receives the first TCP
packet with a valid SYN cookie from the client. Usually, this packet
corresponds to the final step of the TCP 3-way handshake, the ACK
packet. But is also possible to receive a valid SYN cookie from the
first TCP data packet sent by the client, and thus create a child socket
from that SYN cookie.
Since a client socket is ready to send data as soon as it receives the
SYN+ACK packet from the server, the client can send the ACK packet (sent
by the TCP stack code), and the first data packet (sent by the userspace
program) almost at the same time, and thus the server will equally
receive the two TCP packets with valid SYN cookies almost at the same
instant.
When such event happens, the TCP stack code has a race condition that
occurs between the momement a lookup is done to the established
connections hashtable to check for the existence of a connection for the
same client, and the moment that the child socket is added to the
established connections hashtable. As a consequence, this race condition
can lead to a situation where we add two child sockets to the
established connections hashtable and deliver two sockets to the
userspace program to the same client.
This patch fixes the race condition by checking if an existing child
socket exists for the same client when we are adding the second child
socket to the established connections socket. If an existing child
socket exists, we drop the packet and discard the second child socket
to the same client.
Signed-off-by: Ricardo Dias <rdias@singlestore.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20201120111133.GA67501@rdias-suse-pc.lan
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2020-11-20 19:11:33 +08:00
|
|
|
*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
|
|
|
|
&found_dup_sk);
|
2017-10-21 00:04:13 +08:00
|
|
|
if (likely(*own_req)) {
|
2015-11-06 04:50:19 +08:00
|
|
|
tcp_move_syn(newtp, req);
|
2017-10-21 00:04:13 +08:00
|
|
|
ireq->ireq_opt = NULL;
|
|
|
|
} else {
|
tcp: fix race condition when creating child sockets from syncookies
When the TCP stack is in SYN flood mode, the server child socket is
created from the SYN cookie received in a TCP packet with the ACK flag
set.
The child socket is created when the server receives the first TCP
packet with a valid SYN cookie from the client. Usually, this packet
corresponds to the final step of the TCP 3-way handshake, the ACK
packet. But is also possible to receive a valid SYN cookie from the
first TCP data packet sent by the client, and thus create a child socket
from that SYN cookie.
Since a client socket is ready to send data as soon as it receives the
SYN+ACK packet from the server, the client can send the ACK packet (sent
by the TCP stack code), and the first data packet (sent by the userspace
program) almost at the same time, and thus the server will equally
receive the two TCP packets with valid SYN cookies almost at the same
instant.
When such event happens, the TCP stack code has a race condition that
occurs between the momement a lookup is done to the established
connections hashtable to check for the existence of a connection for the
same client, and the moment that the child socket is added to the
established connections hashtable. As a consequence, this race condition
can lead to a situation where we add two child sockets to the
established connections hashtable and deliver two sockets to the
userspace program to the same client.
This patch fixes the race condition by checking if an existing child
socket exists for the same client when we are adding the second child
socket to the established connections socket. If an existing child
socket exists, we drop the packet and discard the second child socket
to the same client.
Signed-off-by: Ricardo Dias <rdias@singlestore.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20201120111133.GA67501@rdias-suse-pc.lan
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2020-11-20 19:11:33 +08:00
|
|
|
if (!req_unhash && found_dup_sk) {
|
|
|
|
/* This code path should only be executed in the
|
|
|
|
* syncookie case only
|
|
|
|
*/
|
|
|
|
bh_unlock_sock(newsk);
|
|
|
|
sock_put(newsk);
|
|
|
|
newsk = NULL;
|
|
|
|
} else {
|
|
|
|
newinet->inet_opt = NULL;
|
|
|
|
}
|
2017-10-21 00:04:13 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
return newsk;
|
|
|
|
|
|
|
|
exit_overflow:
|
2016-04-30 05:16:47 +08:00
|
|
|
NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
|
2010-10-21 19:06:43 +08:00
|
|
|
exit_nonewsk:
|
|
|
|
dst_release(dst);
|
2005-04-17 06:20:36 +08:00
|
|
|
exit:
|
2016-04-01 23:52:20 +08:00
|
|
|
tcp_listendrop(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
return NULL;
|
2011-05-09 06:28:03 +08:00
|
|
|
put_and_exit:
|
2017-10-21 00:04:13 +08:00
|
|
|
newinet->inet_opt = NULL;
|
inet: Fix kmemleak in tcp_v4/6_syn_recv_sock and dccp_v4/6_request_recv_sock
If in either of the above functions inet_csk_route_child_sock() or
__inet_inherit_port() fails, the newsk will not be freed:
unreferenced object 0xffff88022e8a92c0 (size 1592):
comm "softirq", pid 0, jiffies 4294946244 (age 726.160s)
hex dump (first 32 bytes):
0a 01 01 01 0a 01 01 02 00 00 00 00 a7 cc 16 00 ................
02 00 03 01 00 00 00 00 00 00 00 00 00 00 00 00 ................
backtrace:
[<ffffffff8153d190>] kmemleak_alloc+0x21/0x3e
[<ffffffff810ab3e7>] kmem_cache_alloc+0xb5/0xc5
[<ffffffff8149b65b>] sk_prot_alloc.isra.53+0x2b/0xcd
[<ffffffff8149b784>] sk_clone_lock+0x16/0x21e
[<ffffffff814d711a>] inet_csk_clone_lock+0x10/0x7b
[<ffffffff814ebbc3>] tcp_create_openreq_child+0x21/0x481
[<ffffffff814e8fa5>] tcp_v4_syn_recv_sock+0x3a/0x23b
[<ffffffff814ec5ba>] tcp_check_req+0x29f/0x416
[<ffffffff814e8e10>] tcp_v4_do_rcv+0x161/0x2bc
[<ffffffff814eb917>] tcp_v4_rcv+0x6c9/0x701
[<ffffffff814cea9f>] ip_local_deliver_finish+0x70/0xc4
[<ffffffff814cec20>] ip_local_deliver+0x4e/0x7f
[<ffffffff814ce9f8>] ip_rcv_finish+0x1fc/0x233
[<ffffffff814cee68>] ip_rcv+0x217/0x267
[<ffffffff814a7bbe>] __netif_receive_skb+0x49e/0x553
[<ffffffff814a7cc3>] netif_receive_skb+0x50/0x82
This happens, because sk_clone_lock initializes sk_refcnt to 2, and thus
a single sock_put() is not enough to free the memory. Additionally, things
like xfrm, memcg, cookie_values,... may have been initialized.
We have to free them properly.
This is fixed by forcing a call to tcp_done(), ending up in
inet_csk_destroy_sock, doing the final sock_put(). tcp_done() is necessary,
because it ends up doing all the cleanup on xfrm, memcg, cookie_values,
xfrm,...
Before calling tcp_done, we have to set the socket to SOCK_DEAD, to
force it entering inet_csk_destroy_sock. To avoid the warning in
inet_csk_destroy_sock, inet_num has to be set to 0.
As inet_csk_destroy_sock does a dec on orphan_count, we first have to
increase it.
Calling tcp_done() allows us to remove the calls to
tcp_clear_xmit_timer() and tcp_cleanup_congestion_control().
A similar approach is taken for dccp by calling dccp_done().
This is in the kernel since 093d282321 (tproxy: fix hash locking issue
when using port redirection in __inet_inherit_port()), thus since
version >= 2.6.37.
Signed-off-by: Christoph Paasch <christoph.paasch@uclouvain.be>
Signed-off-by: David S. Miller <davem@davemloft.net>
2012-12-14 12:07:58 +08:00
|
|
|
inet_csk_prepare_forced_close(newsk);
|
|
|
|
tcp_done(newsk);
|
2011-05-09 06:28:03 +08:00
|
|
|
goto exit;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-10-03 02:43:32 +08:00
|
|
|
static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2015-10-03 02:43:32 +08:00
|
|
|
#ifdef CONFIG_SYN_COOKIES
|
2015-03-20 10:04:19 +08:00
|
|
|
const struct tcphdr *th = tcp_hdr(skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2010-06-03 08:43:44 +08:00
|
|
|
if (!th->syn)
|
2014-10-16 05:33:22 +08:00
|
|
|
sk = cookie_v4_check(sk, skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
2019-07-30 00:59:14 +08:00
|
|
|
u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
|
|
|
|
struct tcphdr *th, u32 *cookie)
|
|
|
|
{
|
|
|
|
u16 mss = 0;
|
|
|
|
#ifdef CONFIG_SYN_COOKIES
|
|
|
|
mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
|
|
|
|
&tcp_request_sock_ipv4_ops, sk, th);
|
|
|
|
if (mss) {
|
|
|
|
*cookie = __cookie_v4_init_sequence(iph, th, &mss);
|
|
|
|
tcp_synq_overflow(sk);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return mss;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* The socket must have it's spinlock held when we get
|
2015-10-03 02:43:39 +08:00
|
|
|
* here, unless it is a TCP_LISTEN socket.
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* We have a potential double-lock case here, so even when
|
|
|
|
* doing backlog processing we use the BH locking scheme.
|
|
|
|
* This is because we cannot sleep with the original spinlock
|
|
|
|
* held.
|
|
|
|
*/
|
|
|
|
int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
2006-11-15 11:07:45 +08:00
|
|
|
struct sock *rsk;
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
|
2012-07-30 07:20:37 +08:00
|
|
|
struct dst_entry *dst = sk->sk_rx_dst;
|
|
|
|
|
2011-08-15 03:45:55 +08:00
|
|
|
sock_rps_save_rxhash(sk, skb);
|
2014-11-11 21:54:27 +08:00
|
|
|
sk_mark_napi_id(sk, skb);
|
2012-07-30 07:20:37 +08:00
|
|
|
if (dst) {
|
2012-07-27 14:23:40 +08:00
|
|
|
if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||
|
2015-04-03 16:17:26 +08:00
|
|
|
!dst->ops->check(dst, 0)) {
|
2012-07-24 07:29:00 +08:00
|
|
|
dst_release(dst);
|
|
|
|
sk->sk_rx_dst = NULL;
|
|
|
|
}
|
|
|
|
}
|
2018-05-29 23:27:31 +08:00
|
|
|
tcp_rcv_established(sk, skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-06-04 14:49:21 +08:00
|
|
|
if (tcp_checksum_complete(skb))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto csum_err;
|
|
|
|
|
|
|
|
if (sk->sk_state == TCP_LISTEN) {
|
2015-10-03 02:43:32 +08:00
|
|
|
struct sock *nsk = tcp_v4_cookie_check(sk, skb);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!nsk)
|
|
|
|
goto discard;
|
|
|
|
if (nsk != sk) {
|
2006-11-15 11:07:45 +08:00
|
|
|
if (tcp_child_process(sk, nsk, skb)) {
|
|
|
|
rsk = nsk;
|
2005-04-17 06:20:36 +08:00
|
|
|
goto reset;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2010-06-03 17:03:58 +08:00
|
|
|
} else
|
2011-08-15 03:45:55 +08:00
|
|
|
sock_rps_save_rxhash(sk, skb);
|
2010-06-03 17:03:58 +08:00
|
|
|
|
2015-09-29 22:42:41 +08:00
|
|
|
if (tcp_rcv_state_process(sk, skb)) {
|
2006-11-15 11:07:45 +08:00
|
|
|
rsk = sk;
|
2005-04-17 06:20:36 +08:00
|
|
|
goto reset;
|
2006-11-15 11:07:45 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
reset:
|
2006-11-15 11:07:45 +08:00
|
|
|
tcp_v4_send_reset(rsk, skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
discard:
|
|
|
|
kfree_skb(skb);
|
|
|
|
/* Be careful here. If this function gets more complicated and
|
|
|
|
* gcc suffers from register pressure on the x86, sk (in %ebx)
|
|
|
|
* might be destroyed here. This current version compiles correctly,
|
|
|
|
* but you have been warned.
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
csum_err:
|
2016-04-30 05:16:47 +08:00
|
|
|
TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
|
|
|
|
TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
|
2005-04-17 06:20:36 +08:00
|
|
|
goto discard;
|
|
|
|
}
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_v4_do_rcv);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2017-09-28 21:51:36 +08:00
|
|
|
int tcp_v4_early_demux(struct sk_buff *skb)
|
2012-06-20 12:22:05 +08:00
|
|
|
{
|
|
|
|
const struct iphdr *iph;
|
|
|
|
const struct tcphdr *th;
|
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
if (skb->pkt_type != PACKET_HOST)
|
2017-09-28 21:51:36 +08:00
|
|
|
return 0;
|
2012-06-20 12:22:05 +08:00
|
|
|
|
2012-10-23 05:42:47 +08:00
|
|
|
if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
|
2017-09-28 21:51:36 +08:00
|
|
|
return 0;
|
2012-06-20 12:22:05 +08:00
|
|
|
|
|
|
|
iph = ip_hdr(skb);
|
2012-10-23 05:42:47 +08:00
|
|
|
th = tcp_hdr(skb);
|
2012-06-20 12:22:05 +08:00
|
|
|
|
|
|
|
if (th->doff < sizeof(struct tcphdr) / 4)
|
2017-09-28 21:51:36 +08:00
|
|
|
return 0;
|
2012-06-20 12:22:05 +08:00
|
|
|
|
2012-10-23 05:42:47 +08:00
|
|
|
sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
|
2012-06-20 12:22:05 +08:00
|
|
|
iph->saddr, th->source,
|
2012-06-24 01:38:10 +08:00
|
|
|
iph->daddr, ntohs(th->dest),
|
2017-08-07 23:44:17 +08:00
|
|
|
skb->skb_iif, inet_sdif(skb));
|
2012-06-20 12:22:05 +08:00
|
|
|
if (sk) {
|
|
|
|
skb->sk = sk;
|
|
|
|
skb->destructor = sock_edemux;
|
2015-03-16 12:12:13 +08:00
|
|
|
if (sk_fullsock(sk)) {
|
2015-03-23 22:14:00 +08:00
|
|
|
struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
|
2012-07-27 14:23:40 +08:00
|
|
|
|
2012-06-20 12:22:05 +08:00
|
|
|
if (dst)
|
|
|
|
dst = dst_check(dst, 0);
|
2012-07-24 07:29:00 +08:00
|
|
|
if (dst &&
|
2012-07-27 14:23:40 +08:00
|
|
|
inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
|
2012-07-24 07:29:00 +08:00
|
|
|
skb_dst_set_noref(skb, dst);
|
2012-06-20 12:22:05 +08:00
|
|
|
}
|
|
|
|
}
|
2017-09-28 21:51:36 +08:00
|
|
|
return 0;
|
2012-06-20 12:22:05 +08:00
|
|
|
}
|
|
|
|
|
2016-08-27 22:37:54 +08:00
|
|
|
bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
2019-10-10 06:21:13 +08:00
|
|
|
u32 limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf);
|
2018-11-28 06:42:03 +08:00
|
|
|
struct skb_shared_info *shinfo;
|
|
|
|
const struct tcphdr *th;
|
|
|
|
struct tcphdr *thtail;
|
|
|
|
struct sk_buff *tail;
|
|
|
|
unsigned int hdrlen;
|
|
|
|
bool fragstolen;
|
|
|
|
u32 gso_segs;
|
|
|
|
int delta;
|
2016-08-27 22:37:54 +08:00
|
|
|
|
|
|
|
/* In case all data was pulled from skb frags (in __pskb_pull_tail()),
|
|
|
|
* we can fix skb->truesize to its real value to avoid future drops.
|
|
|
|
* This is valid because skb is not yet charged to the socket.
|
|
|
|
* It has been noticed pure SACK packets were sometimes dropped
|
|
|
|
* (if cooked by drivers without copybreak feature).
|
|
|
|
*/
|
2017-01-25 06:57:36 +08:00
|
|
|
skb_condense(skb);
|
2016-08-27 22:37:54 +08:00
|
|
|
|
2018-11-20 09:45:55 +08:00
|
|
|
skb_dst_drop(skb);
|
|
|
|
|
2018-11-28 06:42:03 +08:00
|
|
|
if (unlikely(tcp_checksum_complete(skb))) {
|
|
|
|
bh_unlock_sock(sk);
|
|
|
|
__TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
|
|
|
|
__TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Attempt coalescing to last skb in backlog, even if we are
|
|
|
|
* above the limits.
|
|
|
|
* This is okay because skb capacity is limited to MAX_SKB_FRAGS.
|
|
|
|
*/
|
|
|
|
th = (const struct tcphdr *)skb->data;
|
|
|
|
hdrlen = th->doff * 4;
|
|
|
|
shinfo = skb_shinfo(skb);
|
|
|
|
|
|
|
|
if (!shinfo->gso_size)
|
|
|
|
shinfo->gso_size = skb->len - hdrlen;
|
|
|
|
|
|
|
|
if (!shinfo->gso_segs)
|
|
|
|
shinfo->gso_segs = 1;
|
|
|
|
|
|
|
|
tail = sk->sk_backlog.tail;
|
|
|
|
if (!tail)
|
|
|
|
goto no_coalesce;
|
|
|
|
thtail = (struct tcphdr *)tail->data;
|
|
|
|
|
|
|
|
if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
|
|
|
|
TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
|
|
|
|
((TCP_SKB_CB(tail)->tcp_flags |
|
2019-04-27 01:10:05 +08:00
|
|
|
TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
|
|
|
|
!((TCP_SKB_CB(tail)->tcp_flags &
|
|
|
|
TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
|
2018-11-28 06:42:03 +08:00
|
|
|
((TCP_SKB_CB(tail)->tcp_flags ^
|
|
|
|
TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
|
|
|
|
#ifdef CONFIG_TLS_DEVICE
|
|
|
|
tail->decrypted != skb->decrypted ||
|
|
|
|
#endif
|
|
|
|
thtail->doff != th->doff ||
|
|
|
|
memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
|
|
|
|
goto no_coalesce;
|
|
|
|
|
|
|
|
__skb_pull(skb, hdrlen);
|
|
|
|
if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
|
|
|
|
TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
|
|
|
|
|
tcp: fix receive window update in tcp_add_backlog()
We got reports from GKE customers flows being reset by netfilter
conntrack unless nf_conntrack_tcp_be_liberal is set to 1.
Traces seemed to suggest ACK packet being dropped by the
packet capture, or more likely that ACK were received in the
wrong order.
wscale=7, SYN and SYNACK not shown here.
This ACK allows the sender to send 1871*128 bytes from seq 51359321 :
New right edge of the window -> 51359321+1871*128=51598809
09:17:23.389210 IP A > B: Flags [.], ack 51359321, win 1871, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389212 IP B > A: Flags [.], seq 51422681:51424089, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 1408
09:17:23.389214 IP A > B: Flags [.], ack 51422681, win 1376, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389253 IP B > A: Flags [.], seq 51424089:51488857, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 64768
09:17:23.389272 IP A > B: Flags [.], ack 51488857, win 859, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389275 IP B > A: Flags [.], seq 51488857:51521241, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 32384
Receiver now allows to send 606*128=77568 from seq 51521241 :
New right edge of the window -> 51521241+606*128=51598809
09:17:23.389296 IP A > B: Flags [.], ack 51521241, win 606, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389308 IP B > A: Flags [.], seq 51521241:51553625, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 32384
It seems the sender exceeds RWIN allowance, since 51611353 > 51598809
09:17:23.389346 IP B > A: Flags [.], seq 51553625:51611353, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 57728
09:17:23.389356 IP B > A: Flags [.], seq 51611353:51618393, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 7040
09:17:23.389367 IP A > B: Flags [.], ack 51611353, win 0, options [nop,nop,TS val 10 ecr 999], length 0
netfilter conntrack is not happy and sends RST
09:17:23.389389 IP A > B: Flags [R], seq 92176528, win 0, length 0
09:17:23.389488 IP B > A: Flags [R], seq 174478967, win 0, length 0
Now imagine ACK were delivered out of order and tcp_add_backlog() sets window based on wrong packet.
New right edge of the window -> 51521241+859*128=51631193
Normally TCP stack handles OOO packets just fine, but it
turns out tcp_add_backlog() does not. It can update the window
field of the aggregated packet even if the ACK sequence
of the last received packet is too old.
Many thanks to Alexandre Ferrieux for independently reporting the issue
and suggesting a fix.
Fixes: 4f693b55c3d2 ("tcp: implement coalescing on backlog queue")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Alexandre Ferrieux <alexandre.ferrieux@orange.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-10-05 21:48:13 +08:00
|
|
|
if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
|
2018-11-28 06:42:03 +08:00
|
|
|
TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
|
tcp: fix receive window update in tcp_add_backlog()
We got reports from GKE customers flows being reset by netfilter
conntrack unless nf_conntrack_tcp_be_liberal is set to 1.
Traces seemed to suggest ACK packet being dropped by the
packet capture, or more likely that ACK were received in the
wrong order.
wscale=7, SYN and SYNACK not shown here.
This ACK allows the sender to send 1871*128 bytes from seq 51359321 :
New right edge of the window -> 51359321+1871*128=51598809
09:17:23.389210 IP A > B: Flags [.], ack 51359321, win 1871, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389212 IP B > A: Flags [.], seq 51422681:51424089, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 1408
09:17:23.389214 IP A > B: Flags [.], ack 51422681, win 1376, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389253 IP B > A: Flags [.], seq 51424089:51488857, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 64768
09:17:23.389272 IP A > B: Flags [.], ack 51488857, win 859, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389275 IP B > A: Flags [.], seq 51488857:51521241, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 32384
Receiver now allows to send 606*128=77568 from seq 51521241 :
New right edge of the window -> 51521241+606*128=51598809
09:17:23.389296 IP A > B: Flags [.], ack 51521241, win 606, options [nop,nop,TS val 10 ecr 999], length 0
09:17:23.389308 IP B > A: Flags [.], seq 51521241:51553625, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 32384
It seems the sender exceeds RWIN allowance, since 51611353 > 51598809
09:17:23.389346 IP B > A: Flags [.], seq 51553625:51611353, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 57728
09:17:23.389356 IP B > A: Flags [.], seq 51611353:51618393, ack 1577, win 268, options [nop,nop,TS val 999 ecr 10], length 7040
09:17:23.389367 IP A > B: Flags [.], ack 51611353, win 0, options [nop,nop,TS val 10 ecr 999], length 0
netfilter conntrack is not happy and sends RST
09:17:23.389389 IP A > B: Flags [R], seq 92176528, win 0, length 0
09:17:23.389488 IP B > A: Flags [R], seq 174478967, win 0, length 0
Now imagine ACK were delivered out of order and tcp_add_backlog() sets window based on wrong packet.
New right edge of the window -> 51521241+859*128=51631193
Normally TCP stack handles OOO packets just fine, but it
turns out tcp_add_backlog() does not. It can update the window
field of the aggregated packet even if the ACK sequence
of the last received packet is too old.
Many thanks to Alexandre Ferrieux for independently reporting the issue
and suggesting a fix.
Fixes: 4f693b55c3d2 ("tcp: implement coalescing on backlog queue")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Alexandre Ferrieux <alexandre.ferrieux@orange.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-10-05 21:48:13 +08:00
|
|
|
thtail->window = th->window;
|
|
|
|
}
|
2018-11-28 06:42:03 +08:00
|
|
|
|
2019-04-27 01:10:05 +08:00
|
|
|
/* We have to update both TCP_SKB_CB(tail)->tcp_flags and
|
|
|
|
* thtail->fin, so that the fast path in tcp_rcv_established()
|
|
|
|
* is not entered if we append a packet with a FIN.
|
|
|
|
* SYN, RST, URG are not present.
|
|
|
|
* ACK is set on both packets.
|
|
|
|
* PSH : we do not really care in TCP stack,
|
|
|
|
* at least for 'GRO' packets.
|
|
|
|
*/
|
|
|
|
thtail->fin |= th->fin;
|
2018-11-28 06:42:03 +08:00
|
|
|
TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
|
|
|
|
|
|
|
|
if (TCP_SKB_CB(skb)->has_rxtstamp) {
|
|
|
|
TCP_SKB_CB(tail)->has_rxtstamp = true;
|
|
|
|
tail->tstamp = skb->tstamp;
|
|
|
|
skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Not as strict as GRO. We only need to carry mss max value */
|
|
|
|
skb_shinfo(tail)->gso_size = max(shinfo->gso_size,
|
|
|
|
skb_shinfo(tail)->gso_size);
|
|
|
|
|
|
|
|
gso_segs = skb_shinfo(tail)->gso_segs + shinfo->gso_segs;
|
|
|
|
skb_shinfo(tail)->gso_segs = min_t(u32, gso_segs, 0xFFFF);
|
|
|
|
|
|
|
|
sk->sk_backlog.len += delta;
|
|
|
|
__NET_INC_STATS(sock_net(sk),
|
|
|
|
LINUX_MIB_TCPBACKLOGCOALESCE);
|
|
|
|
kfree_skb_partial(skb, fragstolen);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
__skb_push(skb, hdrlen);
|
|
|
|
|
|
|
|
no_coalesce:
|
|
|
|
/* Only socket owner can try to collapse/prune rx queues
|
|
|
|
* to reduce memory overhead, so add a little headroom here.
|
|
|
|
* Few sockets backlog are possibly concurrently non empty.
|
|
|
|
*/
|
|
|
|
limit += 64*1024;
|
|
|
|
|
2016-08-27 22:37:54 +08:00
|
|
|
if (unlikely(sk_add_backlog(sk, skb, limit))) {
|
|
|
|
bh_unlock_sock(sk);
|
|
|
|
__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(tcp_add_backlog);
|
|
|
|
|
2016-11-11 05:12:35 +08:00
|
|
|
int tcp_filter(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct tcphdr *th = (struct tcphdr *)skb->data;
|
|
|
|
|
2019-03-12 02:41:05 +08:00
|
|
|
return sk_filter_trim_cap(sk, skb, th->doff * 4);
|
2016-11-11 05:12:35 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(tcp_filter);
|
|
|
|
|
2017-12-04 01:32:59 +08:00
|
|
|
static void tcp_v4_restore_cb(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
|
|
|
|
sizeof(struct inet_skb_parm));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
|
|
|
|
const struct tcphdr *th)
|
|
|
|
{
|
|
|
|
/* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
|
|
|
|
* barrier() makes sure compiler wont play fool^Waliasing games.
|
|
|
|
*/
|
|
|
|
memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
|
|
|
|
sizeof(struct inet_skb_parm));
|
|
|
|
barrier();
|
|
|
|
|
|
|
|
TCP_SKB_CB(skb)->seq = ntohl(th->seq);
|
|
|
|
TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
|
|
|
|
skb->len - th->doff * 4);
|
|
|
|
TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
|
|
|
|
TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
|
|
|
|
TCP_SKB_CB(skb)->tcp_tw_isn = 0;
|
|
|
|
TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
|
|
|
|
TCP_SKB_CB(skb)->sacked = 0;
|
|
|
|
TCP_SKB_CB(skb)->has_rxtstamp =
|
|
|
|
skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* From tcp_input.c
|
|
|
|
*/
|
|
|
|
|
|
|
|
int tcp_v4_rcv(struct sk_buff *skb)
|
|
|
|
{
|
2016-04-01 23:52:17 +08:00
|
|
|
struct net *net = dev_net(skb->dev);
|
tcp: add one skb cache for rx
Often times, recvmsg() system calls and BH handling for a particular
TCP socket are done on different cpus.
This means the incoming skb had to be allocated on a cpu,
but freed on another.
This incurs a high spinlock contention in slab layer for small rpc,
but also a high number of cache line ping pongs for larger packets.
A full size GRO packet might use 45 page fragments, meaning
that up to 45 put_page() can be involved.
More over performing the __kfree_skb() in the recvmsg() context
adds a latency for user applications, and increase probability
of trapping them in backlog processing, since the BH handler
might found the socket owned by the user.
This patch, combined with the prior one increases the rpc
performance by about 10 % on servers with large number of cores.
(tcp_rr workload with 10,000 flows and 112 threads reach 9 Mpps
instead of 8 Mpps)
This also increases single bulk flow performance on 40Gbit+ links,
since in this case there are often two cpus working in tandem :
- CPU handling the NIC rx interrupts, feeding the receive queue,
and (after this patch) freeing the skbs that were consumed.
- CPU in recvmsg() system call, essentially 100 % busy copying out
data to user space.
Having at most one skb in a per-socket cache has very little risk
of memory exhaustion, and since it is protected by socket lock,
its management is essentially free.
Note that if rps/rfs is used, we do not enable this feature, because
there is high chance that the same cpu is handling both the recvmsg()
system call and the TCP rx path, but that another cpu did the skb
allocations in the device driver right before the RPS/RFS logic.
To properly handle this case, it seems we would need to record
on which cpu skb was allocated, and use a different channel
to give skbs back to this cpu.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-03-22 23:56:40 +08:00
|
|
|
struct sk_buff *skb_to_free;
|
2017-08-07 23:44:17 +08:00
|
|
|
int sdif = inet_sdif(skb);
|
2019-12-31 06:14:27 +08:00
|
|
|
int dif = inet_iif(skb);
|
2007-04-21 13:47:35 +08:00
|
|
|
const struct iphdr *iph;
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct tcphdr *th;
|
2016-04-01 23:52:17 +08:00
|
|
|
bool refcounted;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sock *sk;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (skb->pkt_type != PACKET_HOST)
|
|
|
|
goto discard_it;
|
|
|
|
|
|
|
|
/* Count it even if it's bad */
|
2016-04-28 07:44:32 +08:00
|
|
|
__TCP_INC_STATS(net, TCP_MIB_INSEGS);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
|
|
|
|
goto discard_it;
|
|
|
|
|
2016-05-14 00:16:40 +08:00
|
|
|
th = (const struct tcphdr *)skb->data;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-05-14 00:16:40 +08:00
|
|
|
if (unlikely(th->doff < sizeof(struct tcphdr) / 4))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto bad_packet;
|
|
|
|
if (!pskb_may_pull(skb, th->doff * 4))
|
|
|
|
goto discard_it;
|
|
|
|
|
|
|
|
/* An explanation is required here, I think.
|
|
|
|
* Packet length and doff are validated by header prediction,
|
2005-11-11 09:13:47 +08:00
|
|
|
* provided case of th->doff==0 is eliminated.
|
2005-04-17 06:20:36 +08:00
|
|
|
* So, we defer the checks. */
|
2014-05-03 07:29:38 +08:00
|
|
|
|
|
|
|
if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
|
2013-04-29 16:39:56 +08:00
|
|
|
goto csum_error;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-05-14 00:16:40 +08:00
|
|
|
th = (const struct tcphdr *)skb->data;
|
2007-04-21 13:47:35 +08:00
|
|
|
iph = ip_hdr(skb);
|
2015-10-14 08:12:54 +08:00
|
|
|
lookup:
|
2016-02-11 00:50:38 +08:00
|
|
|
sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
|
2017-08-07 23:44:17 +08:00
|
|
|
th->dest, sdif, &refcounted);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!sk)
|
|
|
|
goto no_tcp_socket;
|
|
|
|
|
2010-03-09 13:55:56 +08:00
|
|
|
process:
|
|
|
|
if (sk->sk_state == TCP_TIME_WAIT)
|
|
|
|
goto do_time_wait;
|
|
|
|
|
2015-10-03 02:43:32 +08:00
|
|
|
if (sk->sk_state == TCP_NEW_SYN_RECV) {
|
|
|
|
struct request_sock *req = inet_reqsk(sk);
|
2018-02-13 22:14:12 +08:00
|
|
|
bool req_stolen = false;
|
2016-02-18 21:39:18 +08:00
|
|
|
struct sock *nsk;
|
2015-10-03 02:43:32 +08:00
|
|
|
|
|
|
|
sk = req->rsk_listener;
|
2019-12-31 06:14:27 +08:00
|
|
|
if (unlikely(tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))) {
|
2016-08-24 23:50:24 +08:00
|
|
|
sk_drops_add(sk, skb);
|
2016-02-12 14:50:29 +08:00
|
|
|
reqsk_put(req);
|
|
|
|
goto discard_it;
|
|
|
|
}
|
2018-06-13 07:09:37 +08:00
|
|
|
if (tcp_checksum_complete(skb)) {
|
|
|
|
reqsk_put(req);
|
|
|
|
goto csum_error;
|
|
|
|
}
|
2016-02-18 21:39:18 +08:00
|
|
|
if (unlikely(sk->sk_state != TCP_LISTEN)) {
|
2015-10-15 02:16:27 +08:00
|
|
|
inet_csk_reqsk_queue_drop_and_put(sk, req);
|
2015-10-14 08:12:54 +08:00
|
|
|
goto lookup;
|
|
|
|
}
|
2016-04-01 23:52:17 +08:00
|
|
|
/* We own a reference on the listener, increase it again
|
|
|
|
* as we might lose it too soon.
|
|
|
|
*/
|
2016-02-18 21:39:18 +08:00
|
|
|
sock_hold(sk);
|
2016-04-01 23:52:17 +08:00
|
|
|
refcounted = true;
|
2017-09-09 03:44:47 +08:00
|
|
|
nsk = NULL;
|
2017-12-04 01:32:59 +08:00
|
|
|
if (!tcp_filter(sk, skb)) {
|
|
|
|
th = (const struct tcphdr *)skb->data;
|
|
|
|
iph = ip_hdr(skb);
|
|
|
|
tcp_v4_fill_cb(skb, iph, th);
|
2018-02-13 22:14:12 +08:00
|
|
|
nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
|
2017-12-04 01:32:59 +08:00
|
|
|
}
|
2015-10-03 02:43:32 +08:00
|
|
|
if (!nsk) {
|
|
|
|
reqsk_put(req);
|
2018-02-13 22:14:12 +08:00
|
|
|
if (req_stolen) {
|
|
|
|
/* Another cpu got exclusive access to req
|
|
|
|
* and created a full blown socket.
|
|
|
|
* Try to feed this packet to this socket
|
|
|
|
* instead of discarding it.
|
|
|
|
*/
|
|
|
|
tcp_v4_restore_cb(skb);
|
|
|
|
sock_put(sk);
|
|
|
|
goto lookup;
|
|
|
|
}
|
2016-02-18 21:39:18 +08:00
|
|
|
goto discard_and_relse;
|
2015-10-03 02:43:32 +08:00
|
|
|
}
|
|
|
|
if (nsk == sk) {
|
|
|
|
reqsk_put(req);
|
2017-12-04 01:32:59 +08:00
|
|
|
tcp_v4_restore_cb(skb);
|
2015-10-03 02:43:32 +08:00
|
|
|
} else if (tcp_child_process(sk, nsk, skb)) {
|
|
|
|
tcp_v4_send_reset(nsk, skb);
|
2016-02-18 21:39:18 +08:00
|
|
|
goto discard_and_relse;
|
2015-10-03 02:43:32 +08:00
|
|
|
} else {
|
2016-02-18 21:39:18 +08:00
|
|
|
sock_put(sk);
|
2015-10-03 02:43:32 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
2010-03-08 07:21:57 +08:00
|
|
|
if (unlikely(iph->ttl < inet_sk(sk)->min_ttl)) {
|
2016-04-28 07:44:39 +08:00
|
|
|
__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
|
2010-01-12 08:28:01 +08:00
|
|
|
goto discard_and_relse;
|
2010-03-08 07:21:57 +08:00
|
|
|
}
|
2010-01-12 08:28:01 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
|
|
|
|
goto discard_and_relse;
|
2014-08-07 06:38:22 +08:00
|
|
|
|
2019-12-31 06:14:27 +08:00
|
|
|
if (tcp_v4_inbound_md5_hash(sk, skb, dif, sdif))
|
2014-08-07 06:38:22 +08:00
|
|
|
goto discard_and_relse;
|
|
|
|
|
2019-09-30 02:54:03 +08:00
|
|
|
nf_reset_ct(skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-11-11 05:12:35 +08:00
|
|
|
if (tcp_filter(sk, skb))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto discard_and_relse;
|
2016-11-11 05:12:35 +08:00
|
|
|
th = (const struct tcphdr *)skb->data;
|
|
|
|
iph = ip_hdr(skb);
|
2017-12-04 01:32:59 +08:00
|
|
|
tcp_v4_fill_cb(skb, iph, th);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
skb->dev = NULL;
|
|
|
|
|
2015-10-03 02:43:39 +08:00
|
|
|
if (sk->sk_state == TCP_LISTEN) {
|
|
|
|
ret = tcp_v4_do_rcv(sk, skb);
|
|
|
|
goto put_and_return;
|
|
|
|
}
|
|
|
|
|
|
|
|
sk_incoming_cpu_update(sk);
|
|
|
|
|
2006-07-03 15:25:13 +08:00
|
|
|
bh_lock_sock_nested(sk);
|
2016-03-15 01:52:15 +08:00
|
|
|
tcp_segs_in(tcp_sk(sk), skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
ret = 0;
|
|
|
|
if (!sock_owned_by_user(sk)) {
|
tcp: add one skb cache for rx
Often times, recvmsg() system calls and BH handling for a particular
TCP socket are done on different cpus.
This means the incoming skb had to be allocated on a cpu,
but freed on another.
This incurs a high spinlock contention in slab layer for small rpc,
but also a high number of cache line ping pongs for larger packets.
A full size GRO packet might use 45 page fragments, meaning
that up to 45 put_page() can be involved.
More over performing the __kfree_skb() in the recvmsg() context
adds a latency for user applications, and increase probability
of trapping them in backlog processing, since the BH handler
might found the socket owned by the user.
This patch, combined with the prior one increases the rpc
performance by about 10 % on servers with large number of cores.
(tcp_rr workload with 10,000 flows and 112 threads reach 9 Mpps
instead of 8 Mpps)
This also increases single bulk flow performance on 40Gbit+ links,
since in this case there are often two cpus working in tandem :
- CPU handling the NIC rx interrupts, feeding the receive queue,
and (after this patch) freeing the skbs that were consumed.
- CPU in recvmsg() system call, essentially 100 % busy copying out
data to user space.
Having at most one skb in a per-socket cache has very little risk
of memory exhaustion, and since it is protected by socket lock,
its management is essentially free.
Note that if rps/rfs is used, we do not enable this feature, because
there is high chance that the same cpu is handling both the recvmsg()
system call and the TCP rx path, but that another cpu did the skb
allocations in the device driver right before the RPS/RFS logic.
To properly handle this case, it seems we would need to record
on which cpu skb was allocated, and use a different channel
to give skbs back to this cpu.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-03-22 23:56:40 +08:00
|
|
|
skb_to_free = sk->sk_rx_skb_cache;
|
|
|
|
sk->sk_rx_skb_cache = NULL;
|
2017-07-30 09:57:18 +08:00
|
|
|
ret = tcp_v4_do_rcv(sk, skb);
|
tcp: add one skb cache for rx
Often times, recvmsg() system calls and BH handling for a particular
TCP socket are done on different cpus.
This means the incoming skb had to be allocated on a cpu,
but freed on another.
This incurs a high spinlock contention in slab layer for small rpc,
but also a high number of cache line ping pongs for larger packets.
A full size GRO packet might use 45 page fragments, meaning
that up to 45 put_page() can be involved.
More over performing the __kfree_skb() in the recvmsg() context
adds a latency for user applications, and increase probability
of trapping them in backlog processing, since the BH handler
might found the socket owned by the user.
This patch, combined with the prior one increases the rpc
performance by about 10 % on servers with large number of cores.
(tcp_rr workload with 10,000 flows and 112 threads reach 9 Mpps
instead of 8 Mpps)
This also increases single bulk flow performance on 40Gbit+ links,
since in this case there are often two cpus working in tandem :
- CPU handling the NIC rx interrupts, feeding the receive queue,
and (after this patch) freeing the skbs that were consumed.
- CPU in recvmsg() system call, essentially 100 % busy copying out
data to user space.
Having at most one skb in a per-socket cache has very little risk
of memory exhaustion, and since it is protected by socket lock,
its management is essentially free.
Note that if rps/rfs is used, we do not enable this feature, because
there is high chance that the same cpu is handling both the recvmsg()
system call and the TCP rx path, but that another cpu did the skb
allocations in the device driver right before the RPS/RFS logic.
To properly handle this case, it seems we would need to record
on which cpu skb was allocated, and use a different channel
to give skbs back to this cpu.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-03-22 23:56:40 +08:00
|
|
|
} else {
|
|
|
|
if (tcp_add_backlog(sk, skb))
|
|
|
|
goto discard_and_relse;
|
|
|
|
skb_to_free = NULL;
|
2010-03-05 02:01:41 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
bh_unlock_sock(sk);
|
tcp: add one skb cache for rx
Often times, recvmsg() system calls and BH handling for a particular
TCP socket are done on different cpus.
This means the incoming skb had to be allocated on a cpu,
but freed on another.
This incurs a high spinlock contention in slab layer for small rpc,
but also a high number of cache line ping pongs for larger packets.
A full size GRO packet might use 45 page fragments, meaning
that up to 45 put_page() can be involved.
More over performing the __kfree_skb() in the recvmsg() context
adds a latency for user applications, and increase probability
of trapping them in backlog processing, since the BH handler
might found the socket owned by the user.
This patch, combined with the prior one increases the rpc
performance by about 10 % on servers with large number of cores.
(tcp_rr workload with 10,000 flows and 112 threads reach 9 Mpps
instead of 8 Mpps)
This also increases single bulk flow performance on 40Gbit+ links,
since in this case there are often two cpus working in tandem :
- CPU handling the NIC rx interrupts, feeding the receive queue,
and (after this patch) freeing the skbs that were consumed.
- CPU in recvmsg() system call, essentially 100 % busy copying out
data to user space.
Having at most one skb in a per-socket cache has very little risk
of memory exhaustion, and since it is protected by socket lock,
its management is essentially free.
Note that if rps/rfs is used, we do not enable this feature, because
there is high chance that the same cpu is handling both the recvmsg()
system call and the TCP rx path, but that another cpu did the skb
allocations in the device driver right before the RPS/RFS logic.
To properly handle this case, it seems we would need to record
on which cpu skb was allocated, and use a different channel
to give skbs back to this cpu.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-03-22 23:56:40 +08:00
|
|
|
if (skb_to_free)
|
|
|
|
__kfree_skb(skb_to_free);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-10-03 02:43:39 +08:00
|
|
|
put_and_return:
|
2016-04-01 23:52:17 +08:00
|
|
|
if (refcounted)
|
|
|
|
sock_put(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
no_tcp_socket:
|
|
|
|
if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
|
|
|
|
goto discard_it;
|
|
|
|
|
2017-12-04 01:32:59 +08:00
|
|
|
tcp_v4_fill_cb(skb, iph, th);
|
|
|
|
|
2015-06-04 14:49:21 +08:00
|
|
|
if (tcp_checksum_complete(skb)) {
|
2013-04-29 16:39:56 +08:00
|
|
|
csum_error:
|
2016-04-28 07:44:32 +08:00
|
|
|
__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
|
2005-04-17 06:20:36 +08:00
|
|
|
bad_packet:
|
2016-04-28 07:44:32 +08:00
|
|
|
__TCP_INC_STATS(net, TCP_MIB_INERRS);
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
2006-11-15 11:07:45 +08:00
|
|
|
tcp_v4_send_reset(NULL, skb);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
discard_it:
|
|
|
|
/* Discard frame. */
|
|
|
|
kfree_skb(skb);
|
2007-02-09 22:24:47 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
discard_and_relse:
|
2016-04-01 23:52:19 +08:00
|
|
|
sk_drops_add(sk, skb);
|
2016-04-01 23:52:17 +08:00
|
|
|
if (refcounted)
|
|
|
|
sock_put(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
goto discard_it;
|
|
|
|
|
|
|
|
do_time_wait:
|
|
|
|
if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
|
2006-10-11 10:41:46 +08:00
|
|
|
inet_twsk_put(inet_twsk(sk));
|
2005-04-17 06:20:36 +08:00
|
|
|
goto discard_it;
|
|
|
|
}
|
|
|
|
|
2017-12-04 01:32:59 +08:00
|
|
|
tcp_v4_fill_cb(skb, iph, th);
|
|
|
|
|
2013-04-29 16:39:56 +08:00
|
|
|
if (tcp_checksum_complete(skb)) {
|
|
|
|
inet_twsk_put(inet_twsk(sk));
|
|
|
|
goto csum_error;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2006-10-11 10:41:46 +08:00
|
|
|
switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
case TCP_TW_SYN: {
|
2008-03-25 20:47:49 +08:00
|
|
|
struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
|
2016-02-11 00:50:38 +08:00
|
|
|
&tcp_hashinfo, skb,
|
|
|
|
__tcp_hdrlen(th),
|
2013-01-22 17:50:24 +08:00
|
|
|
iph->saddr, th->source,
|
2007-04-21 13:47:35 +08:00
|
|
|
iph->daddr, th->dest,
|
2017-08-07 23:44:17 +08:00
|
|
|
inet_iif(skb),
|
|
|
|
sdif);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (sk2) {
|
2015-07-09 05:28:30 +08:00
|
|
|
inet_twsk_deschedule_put(inet_twsk(sk));
|
2005-04-17 06:20:36 +08:00
|
|
|
sk = sk2;
|
2017-12-04 01:32:59 +08:00
|
|
|
tcp_v4_restore_cb(skb);
|
2016-04-01 23:52:17 +08:00
|
|
|
refcounted = false;
|
2005-04-17 06:20:36 +08:00
|
|
|
goto process;
|
|
|
|
}
|
|
|
|
}
|
2017-10-17 04:48:55 +08:00
|
|
|
/* to ACK */
|
2020-03-13 06:50:22 +08:00
|
|
|
fallthrough;
|
2005-04-17 06:20:36 +08:00
|
|
|
case TCP_TW_ACK:
|
|
|
|
tcp_v4_timewait_ack(sk, skb);
|
|
|
|
break;
|
|
|
|
case TCP_TW_RST:
|
tcp: honour SO_BINDTODEVICE for TW_RST case too
Hannes points out that when we generate tcp reset for timewait sockets we
pretend we found no socket and pass NULL sk to tcp_vX_send_reset().
Make it cope with inet tw sockets and then provide tw sk.
This makes RSTs appear on correct interface when SO_BINDTODEVICE is used.
Packetdrill test case:
// want default route to be used, we rely on BINDTODEVICE
`ip route del 192.0.2.0/24 via 192.168.0.2 dev tun0`
0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3
// test case still works due to BINDTODEVICE
0.001 setsockopt(3, SOL_SOCKET, SO_BINDTODEVICE, "tun0", 4) = 0
0.100...0.200 connect(3, ..., ...) = 0
0.100 > S 0:0(0) <mss 1460,sackOK,nop,nop>
0.200 < S. 0:0(0) ack 1 win 32792 <mss 1460,sackOK,nop,nop>
0.200 > . 1:1(0) ack 1
0.210 close(3) = 0
0.210 > F. 1:1(0) ack 1 win 29200
0.300 < . 1:1(0) ack 2 win 46
// more data while in FIN_WAIT2, expect RST
1.300 < P. 1:1001(1000) ack 1 win 46
// fails without this change -- default route is used
1.301 > R 1:1(0) win 0
Reported-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Florian Westphal <fw@strlen.de>
Acked-by: Eric Dumazet <edumazet@google.com>
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-22 04:29:26 +08:00
|
|
|
tcp_v4_send_reset(sk, skb);
|
|
|
|
inet_twsk_deschedule_put(inet_twsk(sk));
|
|
|
|
goto discard_it;
|
2005-04-17 06:20:36 +08:00
|
|
|
case TCP_TW_SUCCESS:;
|
|
|
|
}
|
|
|
|
goto discard_it;
|
|
|
|
}
|
|
|
|
|
2010-12-02 10:09:13 +08:00
|
|
|
static struct timewait_sock_ops tcp_timewait_sock_ops = {
|
|
|
|
.twsk_obj_size = sizeof(struct tcp_timewait_sock),
|
|
|
|
.twsk_unique = tcp_twsk_unique,
|
|
|
|
.twsk_destructor= tcp_twsk_destructor,
|
|
|
|
};
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-08-09 22:11:00 +08:00
|
|
|
void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
|
2012-08-06 13:09:33 +08:00
|
|
|
{
|
|
|
|
struct dst_entry *dst = skb_dst(skb);
|
|
|
|
|
net: fix IP early demux races
David Wilder reported crashes caused by dst reuse.
<quote David>
I am seeing a crash on a distro V4.2.3 kernel caused by a double
release of a dst_entry. In ipv4_dst_destroy() the call to
list_empty() finds a poisoned next pointer, indicating the dst_entry
has already been removed from the list and freed. The crash occurs
18 to 24 hours into a run of a network stress exerciser.
</quote>
Thanks to his detailed report and analysis, we were able to understand
the core issue.
IP early demux can associate a dst to skb, after a lookup in TCP/UDP
sockets.
When socket cache is not properly set, we want to store into
sk->sk_dst_cache the dst for future IP early demux lookups,
by acquiring a stable refcount on the dst.
Problem is this acquisition is simply using an atomic_inc(),
which works well, unless the dst was queued for destruction from
dst_release() noticing dst refcount went to zero, if DST_NOCACHE
was set on dst.
We need to make sure current refcount is not zero before incrementing
it, or risk double free as David reported.
This patch, being a stable candidate, adds two new helpers, and use
them only from IP early demux problematic paths.
It might be possible to merge in net-next skb_dst_force() and
skb_dst_force_safe(), but I prefer having the smallest patch for stable
kernels : Maybe some skb_dst_force() callers do not expect skb->dst
can suddenly be cleared.
Can probably be backported back to linux-3.6 kernels
Reported-by: David J. Wilder <dwilder@us.ibm.com>
Tested-by: David J. Wilder <dwilder@us.ibm.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-15 06:08:53 +08:00
|
|
|
if (dst && dst_hold_safe(dst)) {
|
2014-09-08 23:06:07 +08:00
|
|
|
sk->sk_rx_dst = dst;
|
|
|
|
inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
|
|
|
|
}
|
2012-08-06 13:09:33 +08:00
|
|
|
}
|
2012-08-09 22:11:00 +08:00
|
|
|
EXPORT_SYMBOL(inet_sk_rx_dst_set);
|
2012-08-06 13:09:33 +08:00
|
|
|
|
2009-09-02 03:25:04 +08:00
|
|
|
const struct inet_connection_sock_af_ops ipv4_specific = {
|
2006-03-21 14:48:35 +08:00
|
|
|
.queue_xmit = ip_queue_xmit,
|
|
|
|
.send_check = tcp_v4_send_check,
|
|
|
|
.rebuild_header = inet_sk_rebuild_header,
|
2012-08-06 13:09:33 +08:00
|
|
|
.sk_rx_dst_set = inet_sk_rx_dst_set,
|
2006-03-21 14:48:35 +08:00
|
|
|
.conn_request = tcp_v4_conn_request,
|
|
|
|
.syn_recv_sock = tcp_v4_syn_recv_sock,
|
|
|
|
.net_header_len = sizeof(struct iphdr),
|
|
|
|
.setsockopt = ip_setsockopt,
|
|
|
|
.getsockopt = ip_getsockopt,
|
|
|
|
.addr2sockaddr = inet_csk_addr2sockaddr,
|
|
|
|
.sockaddr_len = sizeof(struct sockaddr_in),
|
2014-08-15 00:40:05 +08:00
|
|
|
.mtu_reduced = tcp_v4_mtu_reduced,
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(ipv4_specific);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2009-09-02 03:25:03 +08:00
|
|
|
static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
|
2006-11-15 11:07:45 +08:00
|
|
|
.md5_lookup = tcp_v4_md5_lookup,
|
2008-07-19 15:01:42 +08:00
|
|
|
.calc_md5_hash = tcp_v4_md5_hash_skb,
|
2006-11-15 11:07:45 +08:00
|
|
|
.md5_parse = tcp_v4_parse_md5_keys,
|
|
|
|
};
|
2006-12-01 11:16:28 +08:00
|
|
|
#endif
|
2006-11-15 11:07:45 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* NOTE: A lot of things set to zero explicitly by call to
|
|
|
|
* sk_alloc() so need not be done here.
|
|
|
|
*/
|
|
|
|
static int tcp_v4_init_sock(struct sock *sk)
|
|
|
|
{
|
2005-08-10 15:03:31 +08:00
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-04-19 17:55:21 +08:00
|
|
|
tcp_init_sock(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-12-14 15:15:52 +08:00
|
|
|
icsk->icsk_af_ops = &ipv4_specific;
|
2012-04-19 17:55:21 +08:00
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
2012-04-23 15:21:58 +08:00
|
|
|
tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
|
2006-11-15 11:07:45 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-06-15 08:04:49 +08:00
|
|
|
void tcp_v4_destroy_sock(struct sock *sk)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
|
|
|
|
2017-10-24 00:20:26 +08:00
|
|
|
trace_tcp_destroy_sock(sk);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
tcp_clear_xmit_timers(sk);
|
|
|
|
|
2005-08-10 15:03:31 +08:00
|
|
|
tcp_cleanup_congestion_control(sk);
|
2005-06-24 03:19:55 +08:00
|
|
|
|
2017-06-15 02:37:14 +08:00
|
|
|
tcp_cleanup_ulp(sk);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Cleanup up the write buffer. */
|
2007-03-08 04:12:44 +08:00
|
|
|
tcp_write_queue_purge(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
net/tcp_fastopen: Disable active side TFO in certain scenarios
Middlebox firewall issues can potentially cause server's data being
blackholed after a successful 3WHS using TFO. Following are the related
reports from Apple:
https://www.nanog.org/sites/default/files/Paasch_Network_Support.pdf
Slide 31 identifies an issue where the client ACK to the server's data
sent during a TFO'd handshake is dropped.
C ---> syn-data ---> S
C <--- syn/ack ----- S
C (accept & write)
C <---- data ------- S
C ----- ACK -> X S
[retry and timeout]
https://www.ietf.org/proceedings/94/slides/slides-94-tcpm-13.pdf
Slide 5 shows a similar situation that the server's data gets dropped
after 3WHS.
C ---- syn-data ---> S
C <--- syn/ack ----- S
C ---- ack --------> S
S (accept & write)
C? X <- data ------ S
[retry and timeout]
This is the worst failure b/c the client can not detect such behavior to
mitigate the situation (such as disabling TFO). Failing to proceed, the
application (e.g., SSL library) may simply timeout and retry with TFO
again, and the process repeats indefinitely.
The proposed solution is to disable active TFO globally under the
following circumstances:
1. client side TFO socket detects out of order FIN
2. client side TFO socket receives out of order RST
We disable active side TFO globally for 1hr at first. Then if it
happens again, we disable it for 2h, then 4h, 8h, ...
And we reset the timeout to 1hr if a client side TFO sockets not opened
on loopback has successfully received data segs from server.
And we examine this condition during close().
The rational behind it is that when such firewall issue happens,
application running on the client should eventually close the socket as
it is not able to get the data it is expecting. Or application running
on the server should close the socket as it is not able to receive any
response from client.
In both cases, out of order FIN or RST will get received on the client
given that the firewall will not block them as no data are in those
frames.
And we want to disable active TFO globally as it helps if the middle box
is very close to the client and most of the connections are likely to
fail.
Also, add a debug sysctl:
tcp_fastopen_blackhole_detect_timeout_sec:
the initial timeout to use when firewall blackhole issue happens.
This can be set and read.
When setting it to 0, it means to disable the active disable logic.
Signed-off-by: Wei Wang <weiwan@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-04-21 05:45:46 +08:00
|
|
|
/* Check if we want to disable active TFO */
|
|
|
|
tcp_fastopen_active_disable_ofo_check(sk);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Cleans up our, hopefully empty, out_of_order_queue. */
|
tcp: use an RB tree for ooo receive queue
Over the years, TCP BDP has increased by several orders of magnitude,
and some people are considering to reach the 2 Gbytes limit.
Even with current window scale limit of 14, ~1 Gbytes maps to ~740,000
MSS.
In presence of packet losses (or reorders), TCP stores incoming packets
into an out of order queue, and number of skbs sitting there waiting for
the missing packets to be received can be in the 10^5 range.
Most packets are appended to the tail of this queue, and when
packets can finally be transferred to receive queue, we scan the queue
from its head.
However, in presence of heavy losses, we might have to find an arbitrary
point in this queue, involving a linear scan for every incoming packet,
throwing away cpu caches.
This patch converts it to a RB tree, to get bounded latencies.
Yaogong wrote a preliminary patch about 2 years ago.
Eric did the rebase, added ofo_last_skb cache, polishing and tests.
Tested with network dropping between 1 and 10 % packets, with good
success (about 30 % increase of throughput in stress tests)
Next step would be to also use an RB tree for the write queue at sender
side ;)
Signed-off-by: Yaogong Wang <wygivan@google.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Cc: Neal Cardwell <ncardwell@google.com>
Cc: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
Acked-By: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-09-08 05:49:28 +08:00
|
|
|
skb_rbtree_purge(&tp->out_of_order_queue);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-11-15 11:07:45 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
|
|
|
/* Clean up the MD5 key list, if any */
|
|
|
|
if (tp->md5sig_info) {
|
2012-01-31 13:18:33 +08:00
|
|
|
tcp_clear_md5_list(sk);
|
2017-12-22 02:29:10 +08:00
|
|
|
kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
|
2006-11-15 11:07:45 +08:00
|
|
|
tp->md5sig_info = NULL;
|
|
|
|
}
|
|
|
|
#endif
|
2006-05-24 09:05:53 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Clean up a referenced TCP bind bucket. */
|
2005-08-10 11:10:42 +08:00
|
|
|
if (inet_csk(sk)->icsk_bind_hash)
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
inet_put_port(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2019-10-11 11:17:38 +08:00
|
|
|
BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
|
TCPCT part 1d: define TCP cookie option, extend existing struct's
Data structures are carefully composed to require minimal additions.
For example, the struct tcp_options_received cookie_plus variable fits
between existing 16-bit and 8-bit variables, requiring no additional
space (taking alignment into consideration). There are no additions to
tcp_request_sock, and only 1 pointer in tcp_sock.
This is a significantly revised implementation of an earlier (year-old)
patch that no longer applies cleanly, with permission of the original
author (Adam Langley):
http://thread.gmane.org/gmane.linux.network/102586
The principle difference is using a TCP option to carry the cookie nonce,
instead of a user configured offset in the data. This is more flexible and
less subject to user configuration error. Such a cookie option has been
suggested for many years, and is also useful without SYN data, allowing
several related concepts to use the same extension option.
"Re: SYN floods (was: does history repeat itself?)", September 9, 1996.
http://www.merit.net/mail.archives/nanog/1996-09/msg00235.html
"Re: what a new TCP header might look like", May 12, 1998.
ftp://ftp.isi.edu/end2end/end2end-interest-1998.mail
These functions will also be used in subsequent patches that implement
additional features.
Requires:
TCPCT part 1a: add request_values parameter for sending SYNACK
TCPCT part 1b: generate Responder Cookie secret
TCPCT part 1c: sysctl_tcp_cookie_size, socket option TCP_COOKIE_TRANSACTIONS
Signed-off-by: William.Allen.Simpson@gmail.com
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-12-03 02:17:05 +08:00
|
|
|
|
2012-07-19 14:43:09 +08:00
|
|
|
/* If socket is aborted during connect operation */
|
|
|
|
tcp_free_fastopen_req(tp);
|
2017-10-19 02:22:51 +08:00
|
|
|
tcp_fastopen_destroy_cipher(sk);
|
2015-05-04 12:34:46 +08:00
|
|
|
tcp_saved_syn_free(tp);
|
2012-07-19 14:43:09 +08:00
|
|
|
|
2011-12-12 05:47:02 +08:00
|
|
|
sk_sockets_allocated_dec(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(tcp_v4_destroy_sock);
|
|
|
|
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
|
|
/* Proc filesystem TCP sock list dumping. */
|
|
|
|
|
2010-06-07 15:43:42 +08:00
|
|
|
/*
|
|
|
|
* Get next listener socket follow cur. If cur is NULL, get first socket
|
|
|
|
* starting from bucket given in st->bucket; when st->bucket is zero the
|
|
|
|
* very first socket in the hash table is returned.
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
static void *listening_get_next(struct seq_file *seq, void *cur)
|
|
|
|
{
|
2020-06-24 07:08:04 +08:00
|
|
|
struct tcp_seq_afinfo *afinfo;
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2008-04-14 13:11:14 +08:00
|
|
|
struct net *net = seq_file_net(seq);
|
2016-04-01 23:52:17 +08:00
|
|
|
struct inet_listen_hashbucket *ilb;
|
2019-12-14 10:20:41 +08:00
|
|
|
struct hlist_nulls_node *node;
|
2016-04-01 23:52:17 +08:00
|
|
|
struct sock *sk = cur;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2020-06-24 07:08:04 +08:00
|
|
|
if (st->bpf_seq_afinfo)
|
|
|
|
afinfo = st->bpf_seq_afinfo;
|
|
|
|
else
|
|
|
|
afinfo = PDE_DATA(file_inode(seq->file));
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (!sk) {
|
2016-04-01 23:52:17 +08:00
|
|
|
get_head:
|
2010-06-07 15:43:42 +08:00
|
|
|
ilb = &tcp_hashinfo.listening_hash[st->bucket];
|
2016-10-20 12:24:58 +08:00
|
|
|
spin_lock(&ilb->lock);
|
2019-12-14 10:20:41 +08:00
|
|
|
sk = sk_nulls_head(&ilb->nulls_head);
|
2010-06-07 15:43:42 +08:00
|
|
|
st->offset = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
goto get_sk;
|
|
|
|
}
|
2008-11-20 16:40:07 +08:00
|
|
|
ilb = &tcp_hashinfo.listening_hash[st->bucket];
|
2005-04-17 06:20:36 +08:00
|
|
|
++st->num;
|
2010-06-07 15:43:42 +08:00
|
|
|
++st->offset;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2019-12-14 10:20:41 +08:00
|
|
|
sk = sk_nulls_next(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
get_sk:
|
2019-12-14 10:20:41 +08:00
|
|
|
sk_nulls_for_each_from(sk, node) {
|
2010-11-22 11:26:12 +08:00
|
|
|
if (!net_eq(sock_net(sk), net))
|
|
|
|
continue;
|
2020-06-24 07:08:04 +08:00
|
|
|
if (afinfo->family == AF_UNSPEC ||
|
|
|
|
sk->sk_family == afinfo->family)
|
2016-04-01 23:52:17 +08:00
|
|
|
return sk;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2016-10-20 12:24:58 +08:00
|
|
|
spin_unlock(&ilb->lock);
|
2010-06-07 15:43:42 +08:00
|
|
|
st->offset = 0;
|
2016-04-01 23:52:17 +08:00
|
|
|
if (++st->bucket < INET_LHTABLE_SIZE)
|
|
|
|
goto get_head;
|
|
|
|
return NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
|
|
|
|
{
|
2010-06-07 15:43:42 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
|
|
|
void *rc;
|
|
|
|
|
|
|
|
st->bucket = 0;
|
|
|
|
st->offset = 0;
|
|
|
|
rc = listening_get_next(seq, NULL);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
while (rc && *pos) {
|
|
|
|
rc = listening_get_next(seq, rc);
|
|
|
|
--*pos;
|
|
|
|
}
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
static inline bool empty_bucket(const struct tcp_iter_state *st)
|
2008-08-28 16:08:02 +08:00
|
|
|
{
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
|
2008-08-28 16:08:02 +08:00
|
|
|
}
|
|
|
|
|
2010-06-07 15:43:42 +08:00
|
|
|
/*
|
|
|
|
* Get first established socket starting from bucket given in st->bucket.
|
|
|
|
* If st->bucket is zero, the very first socket in the hash is returned.
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
static void *established_get_first(struct seq_file *seq)
|
|
|
|
{
|
2020-06-24 07:08:04 +08:00
|
|
|
struct tcp_seq_afinfo *afinfo;
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2008-04-14 13:11:14 +08:00
|
|
|
struct net *net = seq_file_net(seq);
|
2005-04-17 06:20:36 +08:00
|
|
|
void *rc = NULL;
|
|
|
|
|
2020-06-24 07:08:04 +08:00
|
|
|
if (st->bpf_seq_afinfo)
|
|
|
|
afinfo = st->bpf_seq_afinfo;
|
|
|
|
else
|
|
|
|
afinfo = PDE_DATA(file_inode(seq->file));
|
|
|
|
|
2010-06-07 15:43:42 +08:00
|
|
|
st->offset = 0;
|
|
|
|
for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sock *sk;
|
2008-11-17 11:40:17 +08:00
|
|
|
struct hlist_nulls_node *node;
|
2008-11-21 12:39:09 +08:00
|
|
|
spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-08-28 16:08:02 +08:00
|
|
|
/* Lockless fast path for the common case of empty buckets */
|
|
|
|
if (empty_bucket(st))
|
|
|
|
continue;
|
|
|
|
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_lock_bh(lock);
|
2008-11-17 11:40:17 +08:00
|
|
|
sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
|
2020-06-24 07:08:04 +08:00
|
|
|
if ((afinfo->family != AF_UNSPEC &&
|
|
|
|
sk->sk_family != afinfo->family) ||
|
2008-03-26 02:57:35 +08:00
|
|
|
!net_eq(sock_net(sk), net)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
rc = sk;
|
|
|
|
goto out;
|
|
|
|
}
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_unlock_bh(lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
out:
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *established_get_next(struct seq_file *seq, void *cur)
|
|
|
|
{
|
2020-06-24 07:08:04 +08:00
|
|
|
struct tcp_seq_afinfo *afinfo;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sock *sk = cur;
|
2008-11-17 11:40:17 +08:00
|
|
|
struct hlist_nulls_node *node;
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2008-04-14 13:11:14 +08:00
|
|
|
struct net *net = seq_file_net(seq);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2020-06-24 07:08:04 +08:00
|
|
|
if (st->bpf_seq_afinfo)
|
|
|
|
afinfo = st->bpf_seq_afinfo;
|
|
|
|
else
|
|
|
|
afinfo = PDE_DATA(file_inode(seq->file));
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
++st->num;
|
2010-06-07 15:43:42 +08:00
|
|
|
++st->offset;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
sk = sk_nulls_next(sk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-11-17 11:40:17 +08:00
|
|
|
sk_nulls_for_each_from(sk, node) {
|
2020-06-24 07:08:04 +08:00
|
|
|
if ((afinfo->family == AF_UNSPEC ||
|
|
|
|
sk->sk_family == afinfo->family) &&
|
2018-04-11 15:31:28 +08:00
|
|
|
net_eq(sock_net(sk), net))
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
return sk;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
|
|
|
|
++st->bucket;
|
|
|
|
return established_get_first(seq);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void *established_get_idx(struct seq_file *seq, loff_t pos)
|
|
|
|
{
|
2010-06-07 15:43:42 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
|
|
|
void *rc;
|
|
|
|
|
|
|
|
st->bucket = 0;
|
|
|
|
rc = established_get_first(seq);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
while (rc && pos) {
|
|
|
|
rc = established_get_next(seq, rc);
|
|
|
|
--pos;
|
2006-11-17 20:57:30 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
|
|
|
|
{
|
|
|
|
void *rc;
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
st->state = TCP_SEQ_STATE_LISTENING;
|
|
|
|
rc = listening_get_idx(seq, &pos);
|
|
|
|
|
|
|
|
if (!rc) {
|
|
|
|
st->state = TCP_SEQ_STATE_ESTABLISHED;
|
|
|
|
rc = established_get_idx(seq, pos);
|
|
|
|
}
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2010-06-07 15:43:42 +08:00
|
|
|
static void *tcp_seek_last_pos(struct seq_file *seq)
|
|
|
|
{
|
|
|
|
struct tcp_iter_state *st = seq->private;
|
|
|
|
int offset = st->offset;
|
|
|
|
int orig_num = st->num;
|
|
|
|
void *rc = NULL;
|
|
|
|
|
|
|
|
switch (st->state) {
|
|
|
|
case TCP_SEQ_STATE_LISTENING:
|
|
|
|
if (st->bucket >= INET_LHTABLE_SIZE)
|
|
|
|
break;
|
|
|
|
st->state = TCP_SEQ_STATE_LISTENING;
|
|
|
|
rc = listening_get_next(seq, NULL);
|
|
|
|
while (offset-- && rc)
|
|
|
|
rc = listening_get_next(seq, rc);
|
|
|
|
if (rc)
|
|
|
|
break;
|
|
|
|
st->bucket = 0;
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
st->state = TCP_SEQ_STATE_ESTABLISHED;
|
2020-03-13 06:50:22 +08:00
|
|
|
fallthrough;
|
2010-06-07 15:43:42 +08:00
|
|
|
case TCP_SEQ_STATE_ESTABLISHED:
|
|
|
|
if (st->bucket > tcp_hashinfo.ehash_mask)
|
|
|
|
break;
|
|
|
|
rc = established_get_first(seq);
|
|
|
|
while (offset-- && rc)
|
|
|
|
rc = established_get_next(seq, rc);
|
|
|
|
}
|
|
|
|
|
|
|
|
st->num = orig_num;
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2018-04-11 15:31:28 +08:00
|
|
|
void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2010-06-07 15:43:42 +08:00
|
|
|
void *rc;
|
|
|
|
|
|
|
|
if (*pos && *pos == st->last_pos) {
|
|
|
|
rc = tcp_seek_last_pos(seq);
|
|
|
|
if (rc)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
st->state = TCP_SEQ_STATE_LISTENING;
|
|
|
|
st->num = 0;
|
2010-06-07 15:43:42 +08:00
|
|
|
st->bucket = 0;
|
|
|
|
st->offset = 0;
|
|
|
|
rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
|
|
|
|
|
|
|
|
out:
|
|
|
|
st->last_pos = *pos;
|
|
|
|
return rc;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2018-04-11 15:31:28 +08:00
|
|
|
EXPORT_SYMBOL(tcp_seq_start);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-04-11 15:31:28 +08:00
|
|
|
void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2010-06-07 15:43:42 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2005-04-17 06:20:36 +08:00
|
|
|
void *rc = NULL;
|
|
|
|
|
|
|
|
if (v == SEQ_START_TOKEN) {
|
|
|
|
rc = tcp_get_idx(seq, 0);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (st->state) {
|
|
|
|
case TCP_SEQ_STATE_LISTENING:
|
|
|
|
rc = listening_get_next(seq, v);
|
|
|
|
if (!rc) {
|
|
|
|
st->state = TCP_SEQ_STATE_ESTABLISHED;
|
2010-06-07 15:43:42 +08:00
|
|
|
st->bucket = 0;
|
|
|
|
st->offset = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
rc = established_get_first(seq);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case TCP_SEQ_STATE_ESTABLISHED:
|
|
|
|
rc = established_get_next(seq, v);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
++*pos;
|
2010-06-07 15:43:42 +08:00
|
|
|
st->last_pos = *pos;
|
2005-04-17 06:20:36 +08:00
|
|
|
return rc;
|
|
|
|
}
|
2018-04-11 15:31:28 +08:00
|
|
|
EXPORT_SYMBOL(tcp_seq_next);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-04-11 15:31:28 +08:00
|
|
|
void tcp_seq_stop(struct seq_file *seq, void *v)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st = seq->private;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
switch (st->state) {
|
|
|
|
case TCP_SEQ_STATE_LISTENING:
|
|
|
|
if (v != SEQ_START_TOKEN)
|
2016-10-20 12:24:58 +08:00
|
|
|
spin_unlock(&tcp_hashinfo.listening_hash[st->bucket].lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
case TCP_SEQ_STATE_ESTABLISHED:
|
|
|
|
if (v)
|
2008-11-21 12:39:09 +08:00
|
|
|
spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2018-04-11 15:31:28 +08:00
|
|
|
EXPORT_SYMBOL(tcp_seq_stop);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-03-13 07:44:09 +08:00
|
|
|
static void get_openreq4(const struct request_sock *req,
|
2015-10-03 02:43:30 +08:00
|
|
|
struct seq_file *f, int i)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
[NET] Generalise TCP's struct open_request minisock infrastructure
Kept this first changeset minimal, without changing existing names to
ease peer review.
Basicaly tcp_openreq_alloc now receives the or_calltable, that in turn
has two new members:
->slab, that replaces tcp_openreq_cachep
->obj_size, to inform the size of the openreq descendant for
a specific protocol
The protocol specific fields in struct open_request were moved to a
class hierarchy, with the things that are common to all connection
oriented PF_INET protocols in struct inet_request_sock, the TCP ones
in tcp_request_sock, that is an inet_request_sock, that is an
open_request.
I.e. this uses the same approach used for the struct sock class
hierarchy, with sk_prot indicating if the protocol wants to use the
open_request infrastructure by filling in sk_prot->rsk_prot with an
or_calltable.
Results? Performance is improved and TCP v4 now uses only 64 bytes per
open request minisock, down from 96 without this patch :-)
Next changeset will rename some of the structs, fields and functions
mentioned above, struct or_calltable is way unclear, better name it
struct request_sock_ops, s/struct open_request/struct request_sock/g,
etc.
Signed-off-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2005-06-19 13:46:52 +08:00
|
|
|
const struct inet_request_sock *ireq = inet_rsk(req);
|
inet: get rid of central tcp/dccp listener timer
One of the major issue for TCP is the SYNACK rtx handling,
done by inet_csk_reqsk_queue_prune(), fired by the keepalive
timer of a TCP_LISTEN socket.
This function runs for awful long times, with socket lock held,
meaning that other cpus needing this lock have to spin for hundred of ms.
SYNACK are sent in huge bursts, likely to cause severe drops anyway.
This model was OK 15 years ago when memory was very tight.
We now can afford to have a timer per request sock.
Timer invocations no longer need to lock the listener,
and can be run from all cpus in parallel.
With following patch increasing somaxconn width to 32 bits,
I tested a listener with more than 4 million active request sockets,
and a steady SYNFLOOD of ~200,000 SYN per second.
Host was sending ~830,000 SYNACK per second.
This is ~100 times more what we could achieve before this patch.
Later, we will get rid of the listener hash and use ehash instead.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-03-20 10:04:20 +08:00
|
|
|
long delta = req->rsk_timer.expires - jiffies;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-04-24 16:02:16 +08:00
|
|
|
seq_printf(f, "%4d: %08X:%04X %08X:%04X"
|
2013-11-15 06:31:57 +08:00
|
|
|
" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
|
2005-04-17 06:20:36 +08:00
|
|
|
i,
|
2013-10-10 06:21:29 +08:00
|
|
|
ireq->ir_loc_addr,
|
2015-03-13 07:44:09 +08:00
|
|
|
ireq->ir_num,
|
2013-10-10 06:21:29 +08:00
|
|
|
ireq->ir_rmt_addr,
|
|
|
|
ntohs(ireq->ir_rmt_port),
|
2005-04-17 06:20:36 +08:00
|
|
|
TCP_SYN_RECV,
|
|
|
|
0, 0, /* could print option size, but that is af dependent. */
|
|
|
|
1, /* timers active (only the expire timer) */
|
2012-08-09 05:13:53 +08:00
|
|
|
jiffies_delta_to_clock_t(delta),
|
2012-10-28 07:16:46 +08:00
|
|
|
req->num_timeout,
|
2015-10-03 02:43:30 +08:00
|
|
|
from_kuid_munged(seq_user_ns(f),
|
|
|
|
sock_i_uid(req->rsk_listener)),
|
2005-04-17 06:20:36 +08:00
|
|
|
0, /* non standard timer */
|
|
|
|
0, /* open_requests have no inode */
|
2015-03-13 07:44:09 +08:00
|
|
|
0,
|
2013-11-15 06:31:57 +08:00
|
|
|
req);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2013-11-15 06:31:57 +08:00
|
|
|
static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
int timer_active;
|
|
|
|
unsigned long timer_expires;
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct tcp_sock *tp = tcp_sk(sk);
|
2007-02-22 17:13:58 +08:00
|
|
|
const struct inet_connection_sock *icsk = inet_csk(sk);
|
2011-10-21 17:22:42 +08:00
|
|
|
const struct inet_sock *inet = inet_sk(sk);
|
2015-09-29 22:42:52 +08:00
|
|
|
const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
|
2009-10-15 14:30:45 +08:00
|
|
|
__be32 dest = inet->inet_daddr;
|
|
|
|
__be32 src = inet->inet_rcv_saddr;
|
|
|
|
__u16 destp = ntohs(inet->inet_dport);
|
|
|
|
__u16 srcp = ntohs(inet->inet_sport);
|
2009-12-04 08:06:13 +08:00
|
|
|
int rx_queue;
|
2015-11-13 00:43:18 +08:00
|
|
|
int state;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
tcp: Tail loss probe (TLP)
This patch series implement the Tail loss probe (TLP) algorithm described
in http://tools.ietf.org/html/draft-dukkipati-tcpm-tcp-loss-probe-01. The
first patch implements the basic algorithm.
TLP's goal is to reduce tail latency of short transactions. It achieves
this by converting retransmission timeouts (RTOs) occuring due
to tail losses (losses at end of transactions) into fast recovery.
TLP transmits one packet in two round-trips when a connection is in
Open state and isn't receiving any ACKs. The transmitted packet, aka
loss probe, can be either new or a retransmission. When there is tail
loss, the ACK from a loss probe triggers FACK/early-retransmit based
fast recovery, thus avoiding a costly RTO. In the absence of loss,
there is no change in the connection state.
PTO stands for probe timeout. It is a timer event indicating
that an ACK is overdue and triggers a loss probe packet. The PTO value
is set to max(2*SRTT, 10ms) and is adjusted to account for delayed
ACK timer when there is only one oustanding packet.
TLP Algorithm
On transmission of new data in Open state:
-> packets_out > 1: schedule PTO in max(2*SRTT, 10ms).
-> packets_out == 1: schedule PTO in max(2*RTT, 1.5*RTT + 200ms)
-> PTO = min(PTO, RTO)
Conditions for scheduling PTO:
-> Connection is in Open state.
-> Connection is either cwnd limited or no new data to send.
-> Number of probes per tail loss episode is limited to one.
-> Connection is SACK enabled.
When PTO fires:
new_segment_exists:
-> transmit new segment.
-> packets_out++. cwnd remains same.
no_new_packet:
-> retransmit the last segment.
Its ACK triggers FACK or early retransmit based recovery.
ACK path:
-> rearm RTO at start of ACK processing.
-> reschedule PTO if need be.
In addition, the patch includes a small variation to the Early Retransmit
(ER) algorithm, such that ER and TLP together can in principle recover any
N-degree of tail loss through fast recovery. TLP is controlled by the same
sysctl as ER, tcp_early_retrans sysctl.
tcp_early_retrans==0; disables TLP and ER.
==1; enables RFC5827 ER.
==2; delayed ER.
==3; TLP and delayed ER. [DEFAULT]
==4; TLP only.
The TLP patch series have been extensively tested on Google Web servers.
It is most effective for short Web trasactions, where it reduced RTOs by 15%
and improved HTTP response time (average by 6%, 99th percentile by 10%).
The transmitted probes account for <0.5% of the overall transmissions.
Signed-off-by: Nandita Dukkipati <nanditad@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-11 18:00:43 +08:00
|
|
|
if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
|
2017-01-13 14:11:33 +08:00
|
|
|
icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
|
tcp: Tail loss probe (TLP)
This patch series implement the Tail loss probe (TLP) algorithm described
in http://tools.ietf.org/html/draft-dukkipati-tcpm-tcp-loss-probe-01. The
first patch implements the basic algorithm.
TLP's goal is to reduce tail latency of short transactions. It achieves
this by converting retransmission timeouts (RTOs) occuring due
to tail losses (losses at end of transactions) into fast recovery.
TLP transmits one packet in two round-trips when a connection is in
Open state and isn't receiving any ACKs. The transmitted packet, aka
loss probe, can be either new or a retransmission. When there is tail
loss, the ACK from a loss probe triggers FACK/early-retransmit based
fast recovery, thus avoiding a costly RTO. In the absence of loss,
there is no change in the connection state.
PTO stands for probe timeout. It is a timer event indicating
that an ACK is overdue and triggers a loss probe packet. The PTO value
is set to max(2*SRTT, 10ms) and is adjusted to account for delayed
ACK timer when there is only one oustanding packet.
TLP Algorithm
On transmission of new data in Open state:
-> packets_out > 1: schedule PTO in max(2*SRTT, 10ms).
-> packets_out == 1: schedule PTO in max(2*RTT, 1.5*RTT + 200ms)
-> PTO = min(PTO, RTO)
Conditions for scheduling PTO:
-> Connection is in Open state.
-> Connection is either cwnd limited or no new data to send.
-> Number of probes per tail loss episode is limited to one.
-> Connection is SACK enabled.
When PTO fires:
new_segment_exists:
-> transmit new segment.
-> packets_out++. cwnd remains same.
no_new_packet:
-> retransmit the last segment.
Its ACK triggers FACK or early retransmit based recovery.
ACK path:
-> rearm RTO at start of ACK processing.
-> reschedule PTO if need be.
In addition, the patch includes a small variation to the Early Retransmit
(ER) algorithm, such that ER and TLP together can in principle recover any
N-degree of tail loss through fast recovery. TLP is controlled by the same
sysctl as ER, tcp_early_retrans sysctl.
tcp_early_retrans==0; disables TLP and ER.
==1; enables RFC5827 ER.
==2; delayed ER.
==3; TLP and delayed ER. [DEFAULT]
==4; TLP only.
The TLP patch series have been extensively tested on Google Web servers.
It is most effective for short Web trasactions, where it reduced RTOs by 15%
and improved HTTP response time (average by 6%, 99th percentile by 10%).
The transmitted probes account for <0.5% of the overall transmissions.
Signed-off-by: Nandita Dukkipati <nanditad@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Acked-by: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-03-11 18:00:43 +08:00
|
|
|
icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
|
2005-04-17 06:20:36 +08:00
|
|
|
timer_active = 1;
|
2005-08-10 11:10:42 +08:00
|
|
|
timer_expires = icsk->icsk_timeout;
|
|
|
|
} else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
|
2005-04-17 06:20:36 +08:00
|
|
|
timer_active = 4;
|
2005-08-10 11:10:42 +08:00
|
|
|
timer_expires = icsk->icsk_timeout;
|
2007-02-22 17:13:58 +08:00
|
|
|
} else if (timer_pending(&sk->sk_timer)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
timer_active = 2;
|
2007-02-22 17:13:58 +08:00
|
|
|
timer_expires = sk->sk_timer.expires;
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
|
|
|
timer_active = 0;
|
|
|
|
timer_expires = jiffies;
|
|
|
|
}
|
|
|
|
|
2017-12-20 11:12:52 +08:00
|
|
|
state = inet_sk_state_load(sk);
|
2015-11-13 00:43:18 +08:00
|
|
|
if (state == TCP_LISTEN)
|
2019-11-06 06:11:53 +08:00
|
|
|
rx_queue = READ_ONCE(sk->sk_ack_backlog);
|
2009-12-04 08:06:13 +08:00
|
|
|
else
|
2015-11-13 00:43:18 +08:00
|
|
|
/* Because we don't lock the socket,
|
|
|
|
* we might find a transient negative value.
|
2009-12-04 08:06:13 +08:00
|
|
|
*/
|
2019-10-11 11:17:39 +08:00
|
|
|
rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
|
2019-10-11 11:17:40 +08:00
|
|
|
READ_ONCE(tp->copied_seq), 0);
|
2009-12-04 08:06:13 +08:00
|
|
|
|
2008-04-24 16:02:16 +08:00
|
|
|
seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
|
2013-11-15 06:31:57 +08:00
|
|
|
"%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
|
2015-11-13 00:43:18 +08:00
|
|
|
i, src, srcp, dest, destp, state,
|
2019-10-11 11:17:41 +08:00
|
|
|
READ_ONCE(tp->write_seq) - tp->snd_una,
|
2009-12-04 08:06:13 +08:00
|
|
|
rx_queue,
|
2005-04-17 06:20:36 +08:00
|
|
|
timer_active,
|
2012-08-09 05:13:53 +08:00
|
|
|
jiffies_delta_to_clock_t(timer_expires - jiffies),
|
2005-08-10 11:10:42 +08:00
|
|
|
icsk->icsk_retransmits,
|
2012-05-24 15:10:10 +08:00
|
|
|
from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
|
2005-08-10 15:03:31 +08:00
|
|
|
icsk->icsk_probes_out,
|
2007-02-22 17:13:58 +08:00
|
|
|
sock_i_ino(sk),
|
2017-06-30 18:08:01 +08:00
|
|
|
refcount_read(&sk->sk_refcnt), sk,
|
2008-06-28 11:00:19 +08:00
|
|
|
jiffies_to_clock_t(icsk->icsk_rto),
|
|
|
|
jiffies_to_clock_t(icsk->icsk_ack.ato),
|
2019-01-26 02:53:19 +08:00
|
|
|
(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
|
2005-04-17 06:20:36 +08:00
|
|
|
tp->snd_cwnd,
|
2015-11-13 00:43:18 +08:00
|
|
|
state == TCP_LISTEN ?
|
|
|
|
fastopenq->max_qlen :
|
2013-11-15 06:31:57 +08:00
|
|
|
(tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2011-10-21 17:22:42 +08:00
|
|
|
static void get_timewait4_sock(const struct inet_timewait_sock *tw,
|
2013-11-15 06:31:57 +08:00
|
|
|
struct seq_file *f, int i)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
tcp/dccp: get rid of central timewait timer
Using a timer wheel for timewait sockets was nice ~15 years ago when
memory was expensive and machines had a single processor.
This does not scale, code is ugly and source of huge latencies
(Typically 30 ms have been seen, cpus spinning on death_lock spinlock.)
We can afford to use an extra 64 bytes per timewait sock and spread
timewait load to all cpus to have better behavior.
Tested:
On following test, /proc/sys/net/ipv4/tcp_tw_recycle is set to 1
on the target (lpaa24)
Before patch :
lpaa23:~# ./super_netperf 200 -H lpaa24 -t TCP_CC -l 60 -- -p0,0
419594
lpaa23:~# ./super_netperf 200 -H lpaa24 -t TCP_CC -l 60 -- -p0,0
437171
While test is running, we can observe 25 or even 33 ms latencies.
lpaa24:~# ping -c 1000 -i 0.02 -qn lpaa23
...
1000 packets transmitted, 1000 received, 0% packet loss, time 20601ms
rtt min/avg/max/mdev = 0.020/0.217/25.771/1.535 ms, pipe 2
lpaa24:~# ping -c 1000 -i 0.02 -qn lpaa23
...
1000 packets transmitted, 1000 received, 0% packet loss, time 20702ms
rtt min/avg/max/mdev = 0.019/0.183/33.761/1.441 ms, pipe 2
After patch :
About 90% increase of throughput :
lpaa23:~# ./super_netperf 200 -H lpaa24 -t TCP_CC -l 60 -- -p0,0
810442
lpaa23:~# ./super_netperf 200 -H lpaa24 -t TCP_CC -l 60 -- -p0,0
800992
And latencies are kept to minimal values during this load, even
if network utilization is 90% higher :
lpaa24:~# ping -c 1000 -i 0.02 -qn lpaa23
...
1000 packets transmitted, 1000 received, 0% packet loss, time 19991ms
rtt min/avg/max/mdev = 0.023/0.064/0.360/0.042 ms
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-04-13 09:51:09 +08:00
|
|
|
long delta = tw->tw_timer.expires - jiffies;
|
2006-09-28 09:43:50 +08:00
|
|
|
__be32 dest, src;
|
2005-04-17 06:20:36 +08:00
|
|
|
__u16 destp, srcp;
|
|
|
|
|
|
|
|
dest = tw->tw_daddr;
|
|
|
|
src = tw->tw_rcv_saddr;
|
|
|
|
destp = ntohs(tw->tw_dport);
|
|
|
|
srcp = ntohs(tw->tw_sport);
|
|
|
|
|
2008-04-24 16:02:16 +08:00
|
|
|
seq_printf(f, "%4d: %08X:%04X %08X:%04X"
|
2013-11-15 06:31:57 +08:00
|
|
|
" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
|
2005-04-17 06:20:36 +08:00
|
|
|
i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
|
2012-08-09 05:13:53 +08:00
|
|
|
3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
|
2017-06-30 18:08:01 +08:00
|
|
|
refcount_read(&tw->tw_refcnt), tw);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#define TMPSZ 150
|
|
|
|
|
|
|
|
static int tcp4_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
2008-11-03 18:49:10 +08:00
|
|
|
struct tcp_iter_state *st;
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 15:22:02 +08:00
|
|
|
struct sock *sk = v;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-11-15 06:31:57 +08:00
|
|
|
seq_setwidth(seq, TMPSZ - 1);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (v == SEQ_START_TOKEN) {
|
2013-11-15 06:31:57 +08:00
|
|
|
seq_puts(seq, " sl local_address rem_address st tx_queue "
|
2005-04-17 06:20:36 +08:00
|
|
|
"rx_queue tr tm->when retrnsmt uid timeout "
|
|
|
|
"inode");
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
st = seq->private;
|
|
|
|
|
2015-10-03 02:43:32 +08:00
|
|
|
if (sk->sk_state == TCP_TIME_WAIT)
|
|
|
|
get_timewait4_sock(v, seq, st->num);
|
|
|
|
else if (sk->sk_state == TCP_NEW_SYN_RECV)
|
2015-10-03 02:43:30 +08:00
|
|
|
get_openreq4(v, seq, st->num);
|
2015-10-03 02:43:32 +08:00
|
|
|
else
|
|
|
|
get_tcp4_sock(v, seq, st->num);
|
2005-04-17 06:20:36 +08:00
|
|
|
out:
|
2013-11-15 06:31:57 +08:00
|
|
|
seq_pad(seq, '\n');
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-06-24 07:08:05 +08:00
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
|
|
struct bpf_iter__tcp {
|
|
|
|
__bpf_md_ptr(struct bpf_iter_meta *, meta);
|
|
|
|
__bpf_md_ptr(struct sock_common *, sk_common);
|
|
|
|
uid_t uid __aligned(8);
|
|
|
|
};
|
|
|
|
|
|
|
|
static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
|
|
|
|
struct sock_common *sk_common, uid_t uid)
|
|
|
|
{
|
|
|
|
struct bpf_iter__tcp ctx;
|
|
|
|
|
|
|
|
meta->seq_num--; /* skip SEQ_START_TOKEN */
|
|
|
|
ctx.meta = meta;
|
|
|
|
ctx.sk_common = sk_common;
|
|
|
|
ctx.uid = uid;
|
|
|
|
return bpf_iter_run_prog(prog, &ctx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
struct bpf_iter_meta meta;
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
struct sock *sk = v;
|
|
|
|
uid_t uid;
|
|
|
|
|
|
|
|
if (v == SEQ_START_TOKEN)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (sk->sk_state == TCP_TIME_WAIT) {
|
|
|
|
uid = 0;
|
|
|
|
} else if (sk->sk_state == TCP_NEW_SYN_RECV) {
|
|
|
|
const struct request_sock *req = v;
|
|
|
|
|
|
|
|
uid = from_kuid_munged(seq_user_ns(seq),
|
|
|
|
sock_i_uid(req->rsk_listener));
|
|
|
|
} else {
|
|
|
|
uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
|
|
|
|
}
|
|
|
|
|
|
|
|
meta.seq = seq;
|
|
|
|
prog = bpf_iter_get_info(&meta, false);
|
|
|
|
return tcp_prog_seq_show(prog, &meta, v, uid);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
struct bpf_iter_meta meta;
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
|
|
|
|
if (!v) {
|
|
|
|
meta.seq = seq;
|
|
|
|
prog = bpf_iter_get_info(&meta, true);
|
|
|
|
if (prog)
|
|
|
|
(void)tcp_prog_seq_show(prog, &meta, v, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
tcp_seq_stop(seq, v);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct seq_operations bpf_iter_tcp_seq_ops = {
|
|
|
|
.show = bpf_iter_tcp_seq_show,
|
|
|
|
.start = tcp_seq_start,
|
|
|
|
.next = tcp_seq_next,
|
|
|
|
.stop = bpf_iter_tcp_seq_stop,
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2018-04-11 15:31:28 +08:00
|
|
|
static const struct seq_operations tcp4_seq_ops = {
|
|
|
|
.show = tcp4_seq_show,
|
|
|
|
.start = tcp_seq_start,
|
|
|
|
.next = tcp_seq_next,
|
|
|
|
.stop = tcp_seq_stop,
|
|
|
|
};
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
static struct tcp_seq_afinfo tcp4_seq_afinfo = {
|
|
|
|
.family = AF_INET,
|
|
|
|
};
|
|
|
|
|
2010-01-17 11:35:32 +08:00
|
|
|
static int __net_init tcp4_proc_init_net(struct net *net)
|
2008-03-25 05:56:02 +08:00
|
|
|
{
|
2018-04-11 01:42:55 +08:00
|
|
|
if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
|
|
|
|
sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
|
2018-04-11 15:31:28 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
return 0;
|
2008-03-25 05:56:02 +08:00
|
|
|
}
|
|
|
|
|
2010-01-17 11:35:32 +08:00
|
|
|
static void __net_exit tcp4_proc_exit_net(struct net *net)
|
2008-03-25 05:56:02 +08:00
|
|
|
{
|
2018-04-11 15:31:28 +08:00
|
|
|
remove_proc_entry("tcp", net->proc_net);
|
2008-03-25 05:56:02 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations tcp4_net_ops = {
|
|
|
|
.init = tcp4_proc_init_net,
|
|
|
|
.exit = tcp4_proc_exit_net,
|
|
|
|
};
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
int __init tcp4_proc_init(void)
|
|
|
|
{
|
2008-03-25 05:56:02 +08:00
|
|
|
return register_pernet_subsys(&tcp4_net_ops);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void tcp4_proc_exit(void)
|
|
|
|
{
|
2008-03-25 05:56:02 +08:00
|
|
|
unregister_pernet_subsys(&tcp4_net_ops);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_PROC_FS */
|
|
|
|
|
2020-11-13 23:08:08 +08:00
|
|
|
/* @wake is one when sk_stream_write_space() calls us.
|
|
|
|
* This sends EPOLLOUT only if notsent_bytes is half the limit.
|
|
|
|
* This mimics the strategy used in sock_def_write_space().
|
|
|
|
*/
|
|
|
|
bool tcp_stream_memory_free(const struct sock *sk, int wake)
|
|
|
|
{
|
|
|
|
const struct tcp_sock *tp = tcp_sk(sk);
|
|
|
|
u32 notsent_bytes = READ_ONCE(tp->write_seq) -
|
|
|
|
READ_ONCE(tp->snd_nxt);
|
|
|
|
|
|
|
|
return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(tcp_stream_memory_free);
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct proto tcp_prot = {
|
|
|
|
.name = "TCP",
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.close = tcp_close,
|
2018-03-31 06:08:05 +08:00
|
|
|
.pre_connect = tcp_v4_pre_connect,
|
2005-04-17 06:20:36 +08:00
|
|
|
.connect = tcp_v4_connect,
|
|
|
|
.disconnect = tcp_disconnect,
|
2005-08-10 11:10:42 +08:00
|
|
|
.accept = inet_csk_accept,
|
2005-04-17 06:20:36 +08:00
|
|
|
.ioctl = tcp_ioctl,
|
|
|
|
.init = tcp_v4_init_sock,
|
|
|
|
.destroy = tcp_v4_destroy_sock,
|
|
|
|
.shutdown = tcp_shutdown,
|
|
|
|
.setsockopt = tcp_setsockopt,
|
|
|
|
.getsockopt = tcp_getsockopt,
|
2017-01-09 23:55:12 +08:00
|
|
|
.keepalive = tcp_set_keepalive,
|
2005-04-17 06:20:36 +08:00
|
|
|
.recvmsg = tcp_recvmsg,
|
2010-07-11 04:41:55 +08:00
|
|
|
.sendmsg = tcp_sendmsg,
|
|
|
|
.sendpage = tcp_sendpage,
|
2005-04-17 06:20:36 +08:00
|
|
|
.backlog_rcv = tcp_v4_do_rcv,
|
tcp: TCP Small Queues
This introduce TSQ (TCP Small Queues)
TSQ goal is to reduce number of TCP packets in xmit queues (qdisc &
device queues), to reduce RTT and cwnd bias, part of the bufferbloat
problem.
sk->sk_wmem_alloc not allowed to grow above a given limit,
allowing no more than ~128KB [1] per tcp socket in qdisc/dev layers at a
given time.
TSO packets are sized/capped to half the limit, so that we have two
TSO packets in flight, allowing better bandwidth use.
As a side effect, setting the limit to 40000 automatically reduces the
standard gso max limit (65536) to 40000/2 : It can help to reduce
latencies of high prio packets, having smaller TSO packets.
This means we divert sock_wfree() to a tcp_wfree() handler, to
queue/send following frames when skb_orphan() [2] is called for the
already queued skbs.
Results on my dev machines (tg3/ixgbe nics) are really impressive,
using standard pfifo_fast, and with or without TSO/GSO.
Without reduction of nominal bandwidth, we have reduction of buffering
per bulk sender :
< 1ms on Gbit (instead of 50ms with TSO)
< 8ms on 100Mbit (instead of 132 ms)
I no longer have 4 MBytes backlogged in qdisc by a single netperf
session, and both side socket autotuning no longer use 4 Mbytes.
As skb destructor cannot restart xmit itself ( as qdisc lock might be
taken at this point ), we delegate the work to a tasklet. We use one
tasklest per cpu for performance reasons.
If tasklet finds a socket owned by the user, it sets TSQ_OWNED flag.
This flag is tested in a new protocol method called from release_sock(),
to eventually send new segments.
[1] New /proc/sys/net/ipv4/tcp_limit_output_bytes tunable
[2] skb_orphan() is usually called at TX completion time,
but some drivers call it in their start_xmit() handler.
These drivers should at least use BQL, or else a single TCP
session can still fill the whole NIC TX ring, since TSQ will
have no effect.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Dave Taht <dave.taht@bufferbloat.net>
Cc: Tom Herbert <therbert@google.com>
Cc: Matt Mathis <mattmathis@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Cc: Nandita Dukkipati <nanditad@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2012-07-11 13:50:31 +08:00
|
|
|
.release_cb = tcp_release_cb,
|
[SOCK] proto: Add hashinfo member to struct proto
This way we can remove TCP and DCCP specific versions of
sk->sk_prot->get_port: both v4 and v6 use inet_csk_get_port
sk->sk_prot->hash: inet_hash is directly used, only v6 need
a specific version to deal with mapped sockets
sk->sk_prot->unhash: both v4 and v6 use inet_hash directly
struct inet_connection_sock_af_ops also gets a new member, bind_conflict, so
that inet_csk_get_port can find the per family routine.
Now only the lookup routines receive as a parameter a struct inet_hashtable.
With this we further reuse code, reducing the difference among INET transport
protocols.
Eventually work has to be done on UDP and SCTP to make them share this
infrastructure and get as a bonus inet_diag interfaces so that iproute can be
used with these protocols.
net-2.6/net/ipv4/inet_hashtables.c:
struct proto | +8
struct inet_connection_sock_af_ops | +8
2 structs changed
__inet_hash_nolisten | +18
__inet_hash | -210
inet_put_port | +8
inet_bind_bucket_create | +1
__inet_hash_connect | -8
5 functions changed, 27 bytes added, 218 bytes removed, diff: -191
net-2.6/net/core/sock.c:
proto_seq_show | +3
1 function changed, 3 bytes added, diff: +3
net-2.6/net/ipv4/inet_connection_sock.c:
inet_csk_get_port | +15
1 function changed, 15 bytes added, diff: +15
net-2.6/net/ipv4/tcp.c:
tcp_set_state | -7
1 function changed, 7 bytes removed, diff: -7
net-2.6/net/ipv4/tcp_ipv4.c:
tcp_v4_get_port | -31
tcp_v4_hash | -48
tcp_v4_destroy_sock | -7
tcp_v4_syn_recv_sock | -2
tcp_unhash | -179
5 functions changed, 267 bytes removed, diff: -267
net-2.6/net/ipv6/inet6_hashtables.c:
__inet6_hash | +8
1 function changed, 8 bytes added, diff: +8
net-2.6/net/ipv4/inet_hashtables.c:
inet_unhash | +190
inet_hash | +242
2 functions changed, 432 bytes added, diff: +432
vmlinux:
16 functions changed, 485 bytes added, 492 bytes removed, diff: -7
/home/acme/git/net-2.6/net/ipv6/tcp_ipv6.c:
tcp_v6_get_port | -31
tcp_v6_hash | -7
tcp_v6_syn_recv_sock | -9
3 functions changed, 47 bytes removed, diff: -47
/home/acme/git/net-2.6/net/dccp/proto.c:
dccp_destroy_sock | -7
dccp_unhash | -179
dccp_hash | -49
dccp_set_state | -7
dccp_done | +1
5 functions changed, 1 bytes added, 242 bytes removed, diff: -241
/home/acme/git/net-2.6/net/dccp/ipv4.c:
dccp_v4_get_port | -31
dccp_v4_request_recv_sock | -2
2 functions changed, 33 bytes removed, diff: -33
/home/acme/git/net-2.6/net/dccp/ipv6.c:
dccp_v6_get_port | -31
dccp_v6_hash | -7
dccp_v6_request_recv_sock | +5
3 functions changed, 5 bytes added, 38 bytes removed, diff: -33
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-02-03 20:06:04 +08:00
|
|
|
.hash = inet_hash,
|
|
|
|
.unhash = inet_unhash,
|
|
|
|
.get_port = inet_csk_get_port,
|
2005-04-17 06:20:36 +08:00
|
|
|
.enter_memory_pressure = tcp_enter_memory_pressure,
|
2017-06-08 04:29:12 +08:00
|
|
|
.leave_memory_pressure = tcp_leave_memory_pressure,
|
tcp: TCP_NOTSENT_LOWAT socket option
Idea of this patch is to add optional limitation of number of
unsent bytes in TCP sockets, to reduce usage of kernel memory.
TCP receiver might announce a big window, and TCP sender autotuning
might allow a large amount of bytes in write queue, but this has little
performance impact if a large part of this buffering is wasted :
Write queue needs to be large only to deal with large BDP, not
necessarily to cope with scheduling delays (incoming ACKS make room
for the application to queue more bytes)
For most workloads, using a value of 128 KB or less is OK to give
applications enough time to react to POLLOUT events in time
(or being awaken in a blocking sendmsg())
This patch adds two ways to set the limit :
1) Per socket option TCP_NOTSENT_LOWAT
2) A sysctl (/proc/sys/net/ipv4/tcp_notsent_lowat) for sockets
not using TCP_NOTSENT_LOWAT socket option (or setting a zero value)
Default value being UINT_MAX (0xFFFFFFFF), meaning this has no effect.
This changes poll()/select()/epoll() to report POLLOUT
only if number of unsent bytes is below tp->nosent_lowat
Note this might increase number of sendmsg()/sendfile() calls
when using non blocking sockets,
and increase number of context switches for blocking sockets.
Note this is not related to SO_SNDLOWAT (as SO_SNDLOWAT is
defined as :
Specify the minimum number of bytes in the buffer until
the socket layer will pass the data to the protocol)
Tested:
netperf sessions, and watching /proc/net/protocols "memory" column for TCP
With 200 concurrent netperf -t TCP_STREAM sessions, amount of kernel memory
used by TCP buffers shrinks by ~55 % (20567 pages instead of 45458)
lpq83:~# echo -1 >/proc/sys/net/ipv4/tcp_notsent_lowat
lpq83:~# (super_netperf 200 -t TCP_STREAM -H remote -l 90 &); sleep 60 ; grep TCP /proc/net/protocols
TCPv6 1880 2 45458 no 208 yes ipv6 y y y y y y y y y y y y y n y y y y y
TCP 1696 508 45458 no 208 yes kernel y y y y y y y y y y y y y n y y y y y
lpq83:~# echo 131072 >/proc/sys/net/ipv4/tcp_notsent_lowat
lpq83:~# (super_netperf 200 -t TCP_STREAM -H remote -l 90 &); sleep 60 ; grep TCP /proc/net/protocols
TCPv6 1880 2 20567 no 208 yes ipv6 y y y y y y y y y y y y y n y y y y y
TCP 1696 508 20567 no 208 yes kernel y y y y y y y y y y y y y n y y y y y
Using 128KB has no bad effect on the throughput or cpu usage
of a single flow, although there is an increase of context switches.
A bonus is that we hold socket lock for a shorter amount
of time and should improve latencies of ACK processing.
lpq83:~# echo -1 >/proc/sys/net/ipv4/tcp_notsent_lowat
lpq83:~# perf stat -e context-switches ./netperf -H 7.7.7.84 -t omni -l 20 -c -i10,3
OMNI Send TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 7.7.7.84 () port 0 AF_INET : +/-2.500% @ 99% conf.
Local Remote Local Elapsed Throughput Throughput Local Local Remote Remote Local Remote Service
Send Socket Recv Socket Send Time Units CPU CPU CPU CPU Service Service Demand
Size Size Size (sec) Util Util Util Util Demand Demand Units
Final Final % Method % Method
1651584 6291456 16384 20.00 17447.90 10^6bits/s 3.13 S -1.00 U 0.353 -1.000 usec/KB
Performance counter stats for './netperf -H 7.7.7.84 -t omni -l 20 -c -i10,3':
412,514 context-switches
200.034645535 seconds time elapsed
lpq83:~# echo 131072 >/proc/sys/net/ipv4/tcp_notsent_lowat
lpq83:~# perf stat -e context-switches ./netperf -H 7.7.7.84 -t omni -l 20 -c -i10,3
OMNI Send TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 7.7.7.84 () port 0 AF_INET : +/-2.500% @ 99% conf.
Local Remote Local Elapsed Throughput Throughput Local Local Remote Remote Local Remote Service
Send Socket Recv Socket Send Time Units CPU CPU CPU CPU Service Service Demand
Size Size Size (sec) Util Util Util Util Demand Demand Units
Final Final % Method % Method
1593240 6291456 16384 20.00 17321.16 10^6bits/s 3.35 S -1.00 U 0.381 -1.000 usec/KB
Performance counter stats for './netperf -H 7.7.7.84 -t omni -l 20 -c -i10,3':
2,675,818 context-switches
200.029651391 seconds time elapsed
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Neal Cardwell <ncardwell@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Acked-By: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-07-23 11:27:07 +08:00
|
|
|
.stream_memory_free = tcp_stream_memory_free,
|
2005-04-17 06:20:36 +08:00
|
|
|
.sockets_allocated = &tcp_sockets_allocated,
|
2005-08-10 11:11:41 +08:00
|
|
|
.orphan_count = &tcp_orphan_count,
|
2005-04-17 06:20:36 +08:00
|
|
|
.memory_allocated = &tcp_memory_allocated,
|
|
|
|
.memory_pressure = &tcp_memory_pressure,
|
2013-10-20 07:25:36 +08:00
|
|
|
.sysctl_mem = sysctl_tcp_mem,
|
2017-11-07 16:29:28 +08:00
|
|
|
.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
|
|
|
|
.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
|
2005-04-17 06:20:36 +08:00
|
|
|
.max_header = MAX_TCP_HEADER,
|
|
|
|
.obj_size = sizeof(struct tcp_sock),
|
2017-01-18 18:53:44 +08:00
|
|
|
.slab_flags = SLAB_TYPESAFE_BY_RCU,
|
2005-12-14 15:25:19 +08:00
|
|
|
.twsk_prot = &tcp_timewait_sock_ops,
|
2005-06-19 13:47:21 +08:00
|
|
|
.rsk_prot = &tcp_request_sock_ops,
|
2008-03-23 07:50:58 +08:00
|
|
|
.h.hashinfo = &tcp_hashinfo,
|
2010-07-11 04:41:55 +08:00
|
|
|
.no_autobind = true,
|
2015-12-16 11:30:05 +08:00
|
|
|
.diag_destroy = tcp_abort,
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
2010-07-10 05:22:10 +08:00
|
|
|
EXPORT_SYMBOL(tcp_prot);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-01-30 13:35:05 +08:00
|
|
|
static void __net_exit tcp_sk_exit(struct net *net)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
2019-04-01 16:04:53 +08:00
|
|
|
if (net->ipv4.tcp_congestion_control)
|
2020-01-09 08:35:08 +08:00
|
|
|
bpf_module_put(net->ipv4.tcp_congestion_control,
|
|
|
|
net->ipv4.tcp_congestion_control->owner);
|
2017-11-15 00:25:49 +08:00
|
|
|
|
2015-01-30 13:35:05 +08:00
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
|
|
|
|
free_percpu(net->ipv4.tcp_sk);
|
|
|
|
}
|
|
|
|
|
2008-04-04 05:31:33 +08:00
|
|
|
static int __net_init tcp_sk_init(struct net *net)
|
|
|
|
{
|
2016-12-28 17:52:33 +08:00
|
|
|
int res, cpu, cnt;
|
2015-01-30 13:35:05 +08:00
|
|
|
|
|
|
|
net->ipv4.tcp_sk = alloc_percpu(struct sock *);
|
|
|
|
if (!net->ipv4.tcp_sk)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
|
|
|
|
IPPROTO_TCP, net);
|
|
|
|
if (res)
|
|
|
|
goto fail;
|
2016-04-01 23:52:21 +08:00
|
|
|
sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
|
2018-08-23 04:30:45 +08:00
|
|
|
|
|
|
|
/* Please enforce IP_DF and IPID==0 for RST and
|
|
|
|
* ACK sent in SYN-RECV and TIME-WAIT state.
|
|
|
|
*/
|
|
|
|
inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
|
|
|
|
|
2015-01-30 13:35:05 +08:00
|
|
|
*per_cpu_ptr(net->ipv4.tcp_sk, cpu) = sk;
|
|
|
|
}
|
tcp: add rfc3168, section 6.1.1.1. fallback
This work as a follow-up of commit f7b3bec6f516 ("net: allow setting ecn
via routing table") and adds RFC3168 section 6.1.1.1. fallback for outgoing
ECN connections. In other words, this work adds a retry with a non-ECN
setup SYN packet, as suggested from the RFC on the first timeout:
[...] A host that receives no reply to an ECN-setup SYN within the
normal SYN retransmission timeout interval MAY resend the SYN and
any subsequent SYN retransmissions with CWR and ECE cleared. [...]
Schematic client-side view when assuming the server is in tcp_ecn=2 mode,
that is, Linux default since 2009 via commit 255cac91c3c9 ("tcp: extend
ECN sysctl to allow server-side only ECN"):
1) Normal ECN-capable path:
SYN ECE CWR ----->
<----- SYN ACK ECE
ACK ----->
2) Path with broken middlebox, when client has fallback:
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ----->
<----- SYN ACK
ACK ----->
In case we would not have the fallback implemented, the middlebox drop
point would basically end up as:
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
In any case, it's rather a smaller percentage of sites where there would
occur such additional setup latency: it was found in end of 2014 that ~56%
of IPv4 and 65% of IPv6 servers of Alexa 1 million list would negotiate
ECN (aka tcp_ecn=2 default), 0.42% of these webservers will fail to connect
when trying to negotiate with ECN (tcp_ecn=1) due to timeouts, which the
fallback would mitigate with a slight latency trade-off. Recent related
paper on this topic:
Brian Trammell, Mirja Kühlewind, Damiano Boppart, Iain Learmonth,
Gorry Fairhurst, and Richard Scheffenegger:
"Enabling Internet-Wide Deployment of Explicit Congestion Notification."
Proc. PAM 2015, New York.
http://ecn.ethz.ch/ecn-pam15.pdf
Thus, when net.ipv4.tcp_ecn=1 is being set, the patch will perform RFC3168,
section 6.1.1.1. fallback on timeout. For users explicitly not wanting this
which can be in DC use case, we add a net.ipv4.tcp_ecn_fallback knob that
allows for disabling the fallback.
tp->ecn_flags are not being cleared in tcp_ecn_clear_syn() on output, but
rather we let tcp_ecn_rcv_synack() take that over on input path in case a
SYN ACK ECE was delayed. Thus a spurious SYN retransmission will not prevent
ECN being negotiated eventually in that case.
Reference: https://www.ietf.org/proceedings/92/slides/slides-92-iccrg-1.pdf
Reference: https://www.ietf.org/proceedings/89/slides/slides-89-tsvarea-1.pdf
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Mirja Kühlewind <mirja.kuehlewind@tik.ee.ethz.ch>
Signed-off-by: Brian Trammell <trammell@tik.ee.ethz.ch>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Dave That <dave.taht@gmail.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-20 03:04:22 +08:00
|
|
|
|
2013-01-06 00:10:48 +08:00
|
|
|
net->ipv4.sysctl_tcp_ecn = 2;
|
tcp: add rfc3168, section 6.1.1.1. fallback
This work as a follow-up of commit f7b3bec6f516 ("net: allow setting ecn
via routing table") and adds RFC3168 section 6.1.1.1. fallback for outgoing
ECN connections. In other words, this work adds a retry with a non-ECN
setup SYN packet, as suggested from the RFC on the first timeout:
[...] A host that receives no reply to an ECN-setup SYN within the
normal SYN retransmission timeout interval MAY resend the SYN and
any subsequent SYN retransmissions with CWR and ECE cleared. [...]
Schematic client-side view when assuming the server is in tcp_ecn=2 mode,
that is, Linux default since 2009 via commit 255cac91c3c9 ("tcp: extend
ECN sysctl to allow server-side only ECN"):
1) Normal ECN-capable path:
SYN ECE CWR ----->
<----- SYN ACK ECE
ACK ----->
2) Path with broken middlebox, when client has fallback:
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ----->
<----- SYN ACK
ACK ----->
In case we would not have the fallback implemented, the middlebox drop
point would basically end up as:
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
In any case, it's rather a smaller percentage of sites where there would
occur such additional setup latency: it was found in end of 2014 that ~56%
of IPv4 and 65% of IPv6 servers of Alexa 1 million list would negotiate
ECN (aka tcp_ecn=2 default), 0.42% of these webservers will fail to connect
when trying to negotiate with ECN (tcp_ecn=1) due to timeouts, which the
fallback would mitigate with a slight latency trade-off. Recent related
paper on this topic:
Brian Trammell, Mirja Kühlewind, Damiano Boppart, Iain Learmonth,
Gorry Fairhurst, and Richard Scheffenegger:
"Enabling Internet-Wide Deployment of Explicit Congestion Notification."
Proc. PAM 2015, New York.
http://ecn.ethz.ch/ecn-pam15.pdf
Thus, when net.ipv4.tcp_ecn=1 is being set, the patch will perform RFC3168,
section 6.1.1.1. fallback on timeout. For users explicitly not wanting this
which can be in DC use case, we add a net.ipv4.tcp_ecn_fallback knob that
allows for disabling the fallback.
tp->ecn_flags are not being cleared in tcp_ecn_clear_syn() on output, but
rather we let tcp_ecn_rcv_synack() take that over on input path in case a
SYN ACK ECE was delayed. Thus a spurious SYN retransmission will not prevent
ECN being negotiated eventually in that case.
Reference: https://www.ietf.org/proceedings/92/slides/slides-92-iccrg-1.pdf
Reference: https://www.ietf.org/proceedings/89/slides/slides-89-tsvarea-1.pdf
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Mirja Kühlewind <mirja.kuehlewind@tik.ee.ethz.ch>
Signed-off-by: Brian Trammell <trammell@tik.ee.ethz.ch>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Dave That <dave.taht@gmail.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-20 03:04:22 +08:00
|
|
|
net->ipv4.sysctl_tcp_ecn_fallback = 1;
|
|
|
|
|
2015-02-10 09:53:16 +08:00
|
|
|
net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
|
2019-06-07 00:15:31 +08:00
|
|
|
net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
|
2015-03-06 11:18:23 +08:00
|
|
|
net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
|
2015-03-06 11:18:24 +08:00
|
|
|
net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
|
2019-08-08 07:52:29 +08:00
|
|
|
net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
|
2008-04-04 05:31:33 +08:00
|
|
|
|
2016-01-07 22:38:43 +08:00
|
|
|
net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
|
2016-01-07 22:38:44 +08:00
|
|
|
net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
|
2016-01-07 22:38:45 +08:00
|
|
|
net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
|
2016-01-07 22:38:43 +08:00
|
|
|
|
2016-02-03 15:46:49 +08:00
|
|
|
net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
|
2016-02-03 15:46:50 +08:00
|
|
|
net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
|
2016-02-08 17:24:33 +08:00
|
|
|
net->ipv4.sysctl_tcp_syncookies = 1;
|
2016-02-03 15:46:52 +08:00
|
|
|
net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
|
2016-02-03 15:46:53 +08:00
|
|
|
net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
|
2016-02-03 15:46:54 +08:00
|
|
|
net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
|
2016-02-03 15:46:55 +08:00
|
|
|
net->ipv4.sysctl_tcp_orphan_retries = 0;
|
2016-02-03 15:46:56 +08:00
|
|
|
net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
|
2016-02-03 15:46:57 +08:00
|
|
|
net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
|
2018-06-04 01:41:17 +08:00
|
|
|
net->ipv4.sysctl_tcp_tw_reuse = 2;
|
2019-12-10 03:19:59 +08:00
|
|
|
net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
|
2016-02-03 15:46:51 +08:00
|
|
|
|
2016-12-28 17:52:33 +08:00
|
|
|
cnt = tcp_hashinfo.ehash_mask + 1;
|
2018-09-01 20:21:05 +08:00
|
|
|
net->ipv4.tcp_death_row.sysctl_max_tw_buckets = cnt / 2;
|
2016-12-28 17:52:32 +08:00
|
|
|
net->ipv4.tcp_death_row.hashinfo = &tcp_hashinfo;
|
|
|
|
|
2019-10-31 01:05:46 +08:00
|
|
|
net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
|
2017-06-08 01:34:37 +08:00
|
|
|
net->ipv4.sysctl_tcp_sack = 1;
|
2017-06-08 01:34:38 +08:00
|
|
|
net->ipv4.sysctl_tcp_window_scaling = 1;
|
2017-06-08 01:34:39 +08:00
|
|
|
net->ipv4.sysctl_tcp_timestamps = 1;
|
2017-10-27 12:54:56 +08:00
|
|
|
net->ipv4.sysctl_tcp_early_retrans = 3;
|
2017-10-27 12:54:57 +08:00
|
|
|
net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
|
2017-10-27 12:54:59 +08:00
|
|
|
net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
|
2017-10-27 12:55:00 +08:00
|
|
|
net->ipv4.sysctl_tcp_retrans_collapse = 1;
|
2017-10-27 12:55:06 +08:00
|
|
|
net->ipv4.sysctl_tcp_max_reordering = 300;
|
2017-10-27 12:55:07 +08:00
|
|
|
net->ipv4.sysctl_tcp_dsack = 1;
|
2017-10-27 12:55:08 +08:00
|
|
|
net->ipv4.sysctl_tcp_app_win = 31;
|
2017-10-27 12:55:09 +08:00
|
|
|
net->ipv4.sysctl_tcp_adv_win_scale = 1;
|
2017-10-27 12:55:10 +08:00
|
|
|
net->ipv4.sysctl_tcp_frto = 2;
|
2017-10-27 22:47:22 +08:00
|
|
|
net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
|
2017-10-27 22:47:23 +08:00
|
|
|
/* This limits the percentage of the congestion window which we
|
|
|
|
* will allow a single TSO frame to consume. Building TSO frames
|
|
|
|
* which are too large can cause TCP streams to be bursty.
|
|
|
|
*/
|
|
|
|
net->ipv4.sysctl_tcp_tso_win_divisor = 3;
|
2018-11-11 23:34:28 +08:00
|
|
|
/* Default TSQ limit of 16 TSO segments */
|
|
|
|
net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
|
2017-10-27 22:47:26 +08:00
|
|
|
/* rfc5961 challenge ack rate limiting */
|
|
|
|
net->ipv4.sysctl_tcp_challenge_ack_limit = 1000;
|
2017-10-27 22:47:27 +08:00
|
|
|
net->ipv4.sysctl_tcp_min_tso_segs = 2;
|
2017-10-27 22:47:28 +08:00
|
|
|
net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
|
2017-10-27 22:47:29 +08:00
|
|
|
net->ipv4.sysctl_tcp_autocorking = 1;
|
2017-10-27 22:47:30 +08:00
|
|
|
net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
|
2017-10-27 22:47:31 +08:00
|
|
|
net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
|
2017-10-27 22:47:32 +08:00
|
|
|
net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
|
2017-11-07 16:29:28 +08:00
|
|
|
if (net != &init_net) {
|
|
|
|
memcpy(net->ipv4.sysctl_tcp_rmem,
|
|
|
|
init_net.ipv4.sysctl_tcp_rmem,
|
|
|
|
sizeof(init_net.ipv4.sysctl_tcp_rmem));
|
|
|
|
memcpy(net->ipv4.sysctl_tcp_wmem,
|
|
|
|
init_net.ipv4.sysctl_tcp_wmem,
|
|
|
|
sizeof(init_net.ipv4.sysctl_tcp_wmem));
|
|
|
|
}
|
2018-05-18 05:47:28 +08:00
|
|
|
net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
|
2020-05-01 01:35:43 +08:00
|
|
|
net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
|
2018-05-18 05:47:29 +08:00
|
|
|
net->ipv4.sysctl_tcp_comp_sack_nr = 44;
|
2017-09-27 11:35:40 +08:00
|
|
|
net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
|
2017-09-27 11:35:42 +08:00
|
|
|
spin_lock_init(&net->ipv4.tcp_fastopen_ctx_lock);
|
2017-09-27 11:35:43 +08:00
|
|
|
net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 60 * 60;
|
|
|
|
atomic_set(&net->ipv4.tfo_active_disable_times, 0);
|
2017-09-27 11:35:40 +08:00
|
|
|
|
2017-11-15 00:25:49 +08:00
|
|
|
/* Reno is always built in */
|
|
|
|
if (!net_eq(net, &init_net) &&
|
2020-01-09 08:35:08 +08:00
|
|
|
bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
|
|
|
|
init_net.ipv4.tcp_congestion_control->owner))
|
2017-11-15 00:25:49 +08:00
|
|
|
net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
|
|
|
|
else
|
|
|
|
net->ipv4.tcp_congestion_control = &tcp_reno;
|
|
|
|
|
tcp: add rfc3168, section 6.1.1.1. fallback
This work as a follow-up of commit f7b3bec6f516 ("net: allow setting ecn
via routing table") and adds RFC3168 section 6.1.1.1. fallback for outgoing
ECN connections. In other words, this work adds a retry with a non-ECN
setup SYN packet, as suggested from the RFC on the first timeout:
[...] A host that receives no reply to an ECN-setup SYN within the
normal SYN retransmission timeout interval MAY resend the SYN and
any subsequent SYN retransmissions with CWR and ECE cleared. [...]
Schematic client-side view when assuming the server is in tcp_ecn=2 mode,
that is, Linux default since 2009 via commit 255cac91c3c9 ("tcp: extend
ECN sysctl to allow server-side only ECN"):
1) Normal ECN-capable path:
SYN ECE CWR ----->
<----- SYN ACK ECE
ACK ----->
2) Path with broken middlebox, when client has fallback:
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ----->
<----- SYN ACK
ACK ----->
In case we would not have the fallback implemented, the middlebox drop
point would basically end up as:
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
SYN ECE CWR ----X crappy middlebox drops packet
(timeout, rtx)
In any case, it's rather a smaller percentage of sites where there would
occur such additional setup latency: it was found in end of 2014 that ~56%
of IPv4 and 65% of IPv6 servers of Alexa 1 million list would negotiate
ECN (aka tcp_ecn=2 default), 0.42% of these webservers will fail to connect
when trying to negotiate with ECN (tcp_ecn=1) due to timeouts, which the
fallback would mitigate with a slight latency trade-off. Recent related
paper on this topic:
Brian Trammell, Mirja Kühlewind, Damiano Boppart, Iain Learmonth,
Gorry Fairhurst, and Richard Scheffenegger:
"Enabling Internet-Wide Deployment of Explicit Congestion Notification."
Proc. PAM 2015, New York.
http://ecn.ethz.ch/ecn-pam15.pdf
Thus, when net.ipv4.tcp_ecn=1 is being set, the patch will perform RFC3168,
section 6.1.1.1. fallback on timeout. For users explicitly not wanting this
which can be in DC use case, we add a net.ipv4.tcp_ecn_fallback knob that
allows for disabling the fallback.
tp->ecn_flags are not being cleared in tcp_ecn_clear_syn() on output, but
rather we let tcp_ecn_rcv_synack() take that over on input path in case a
SYN ACK ECE was delayed. Thus a spurious SYN retransmission will not prevent
ECN being negotiated eventually in that case.
Reference: https://www.ietf.org/proceedings/92/slides/slides-92-iccrg-1.pdf
Reference: https://www.ietf.org/proceedings/89/slides/slides-89-tsvarea-1.pdf
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Mirja Kühlewind <mirja.kuehlewind@tik.ee.ethz.ch>
Signed-off-by: Brian Trammell <trammell@tik.ee.ethz.ch>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Dave That <dave.taht@gmail.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-05-20 03:04:22 +08:00
|
|
|
return 0;
|
2015-01-30 13:35:05 +08:00
|
|
|
fail:
|
|
|
|
tcp_sk_exit(net);
|
|
|
|
|
|
|
|
return res;
|
2009-12-03 10:29:09 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
|
|
|
|
{
|
2017-09-27 11:35:42 +08:00
|
|
|
struct net *net;
|
|
|
|
|
2016-12-28 17:52:32 +08:00
|
|
|
inet_twsk_purge(&tcp_hashinfo, AF_INET);
|
2017-09-27 11:35:42 +08:00
|
|
|
|
|
|
|
list_for_each_entry(net, net_exit_list, exit_list)
|
|
|
|
tcp_fastopen_ctx_destroy(net);
|
2008-04-04 05:31:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations __net_initdata tcp_sk_ops = {
|
2009-12-03 10:29:09 +08:00
|
|
|
.init = tcp_sk_init,
|
|
|
|
.exit = tcp_sk_exit,
|
|
|
|
.exit_batch = tcp_sk_exit_batch,
|
2008-04-04 05:31:33 +08:00
|
|
|
};
|
|
|
|
|
2020-06-24 07:08:05 +08:00
|
|
|
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
|
|
|
DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
|
|
|
|
struct sock_common *sk_common, uid_t uid)
|
|
|
|
|
2020-07-24 02:41:10 +08:00
|
|
|
static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
|
2020-06-24 07:08:05 +08:00
|
|
|
{
|
|
|
|
struct tcp_iter_state *st = priv_data;
|
|
|
|
struct tcp_seq_afinfo *afinfo;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
afinfo = kmalloc(sizeof(*afinfo), GFP_USER | __GFP_NOWARN);
|
|
|
|
if (!afinfo)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
afinfo->family = AF_UNSPEC;
|
|
|
|
st->bpf_seq_afinfo = afinfo;
|
2020-07-24 02:41:10 +08:00
|
|
|
ret = bpf_iter_init_seq_net(priv_data, aux);
|
2020-06-24 07:08:05 +08:00
|
|
|
if (ret)
|
|
|
|
kfree(afinfo);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void bpf_iter_fini_tcp(void *priv_data)
|
|
|
|
{
|
|
|
|
struct tcp_iter_state *st = priv_data;
|
|
|
|
|
|
|
|
kfree(st->bpf_seq_afinfo);
|
|
|
|
bpf_iter_fini_seq_net(priv_data);
|
|
|
|
}
|
|
|
|
|
2020-07-24 02:41:09 +08:00
|
|
|
static const struct bpf_iter_seq_info tcp_seq_info = {
|
2020-06-24 07:08:05 +08:00
|
|
|
.seq_ops = &bpf_iter_tcp_seq_ops,
|
|
|
|
.init_seq_private = bpf_iter_init_tcp,
|
|
|
|
.fini_seq_private = bpf_iter_fini_tcp,
|
|
|
|
.seq_priv_size = sizeof(struct tcp_iter_state),
|
2020-07-24 02:41:09 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
static struct bpf_iter_reg tcp_reg_info = {
|
|
|
|
.target = "tcp",
|
2020-06-24 07:08:05 +08:00
|
|
|
.ctx_arg_info_size = 1,
|
|
|
|
.ctx_arg_info = {
|
|
|
|
{ offsetof(struct bpf_iter__tcp, sk_common),
|
|
|
|
PTR_TO_BTF_ID_OR_NULL },
|
|
|
|
},
|
2020-07-24 02:41:09 +08:00
|
|
|
.seq_info = &tcp_seq_info,
|
2020-06-24 07:08:05 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
static void __init bpf_iter_register(void)
|
|
|
|
{
|
2020-07-21 00:34:03 +08:00
|
|
|
tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
|
2020-06-24 07:08:05 +08:00
|
|
|
if (bpf_iter_reg_target(&tcp_reg_info))
|
|
|
|
pr_warn("Warning: could not register bpf iterator tcp\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2008-03-01 03:13:15 +08:00
|
|
|
void __init tcp_v4_init(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2009-02-22 16:10:18 +08:00
|
|
|
if (register_pernet_subsys(&tcp_sk_ops))
|
2005-04-17 06:20:36 +08:00
|
|
|
panic("Failed to create the TCP control socket.\n");
|
2020-06-24 07:08:05 +08:00
|
|
|
|
|
|
|
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
|
|
|
bpf_iter_register();
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|