2020-01-22 08:56:15 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/* Multipath TCP
|
|
|
|
*
|
|
|
|
* Copyright (c) 2017 - 2019, Intel Corporation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) "MPTCP: " fmt
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/netdevice.h>
|
2020-01-22 08:56:26 +08:00
|
|
|
#include <linux/sched/signal.h>
|
|
|
|
#include <linux/atomic.h>
|
2020-01-22 08:56:15 +08:00
|
|
|
#include <net/sock.h>
|
|
|
|
#include <net/inet_common.h>
|
|
|
|
#include <net/inet_hashtables.h>
|
|
|
|
#include <net/protocol.h>
|
|
|
|
#include <net/tcp.h>
|
2020-07-29 06:12:03 +08:00
|
|
|
#include <net/tcp_states.h>
|
2020-01-22 08:56:19 +08:00
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
|
|
|
#include <net/transp_v6.h>
|
|
|
|
#endif
|
2020-01-22 08:56:15 +08:00
|
|
|
#include <net/mptcp.h>
|
|
|
|
#include "protocol.h"
|
2020-03-28 05:48:50 +08:00
|
|
|
#include "mib.h"
|
2020-01-22 08:56:15 +08:00
|
|
|
|
2020-02-06 07:39:37 +08:00
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
|
|
|
struct mptcp6_sock {
|
|
|
|
struct mptcp_sock msk;
|
|
|
|
struct ipv6_pinfo np;
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
struct mptcp_skb_cb {
|
2020-09-14 16:01:12 +08:00
|
|
|
u64 map_seq;
|
|
|
|
u64 end_seq;
|
2020-02-26 17:14:48 +08:00
|
|
|
u32 offset;
|
|
|
|
};
|
|
|
|
|
|
|
|
#define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0]))
|
|
|
|
|
2020-03-28 05:48:45 +08:00
|
|
|
static struct percpu_counter mptcp_sockets_allocated;
|
|
|
|
|
2020-01-22 08:56:17 +08:00
|
|
|
/* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not
|
|
|
|
* completed yet or has failed, return the subflow socket.
|
|
|
|
* Otherwise return NULL.
|
|
|
|
*/
|
|
|
|
static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
|
|
|
|
{
|
2020-01-22 08:56:32 +08:00
|
|
|
if (!msk->subflow || READ_ONCE(msk->can_ack))
|
2020-01-22 08:56:17 +08:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return msk->subflow;
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:22 +08:00
|
|
|
static bool mptcp_is_tcpsk(struct sock *sk)
|
2020-04-02 19:44:51 +08:00
|
|
|
{
|
|
|
|
struct socket *sock = sk->sk_socket;
|
|
|
|
|
|
|
|
if (unlikely(sk->sk_prot == &tcp_prot)) {
|
|
|
|
/* we are being invoked after mptcp_accept() has
|
|
|
|
* accepted a non-mp-capable flow: sk is a tcp_sk,
|
|
|
|
* not an mptcp one.
|
|
|
|
*
|
|
|
|
* Hand the socket over to tcp so all further socket ops
|
|
|
|
* bypass mptcp.
|
|
|
|
*/
|
|
|
|
sock->ops = &inet_stream_ops;
|
2020-06-30 04:26:22 +08:00
|
|
|
return true;
|
2020-04-02 19:44:51 +08:00
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
|
|
|
} else if (unlikely(sk->sk_prot == &tcpv6_prot)) {
|
|
|
|
sock->ops = &inet6_stream_ops;
|
2020-06-30 04:26:22 +08:00
|
|
|
return true;
|
2020-04-02 19:44:51 +08:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:22 +08:00
|
|
|
return false;
|
2020-04-02 19:44:51 +08:00
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:24 +08:00
|
|
|
static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk)
|
2020-01-22 08:56:18 +08:00
|
|
|
{
|
|
|
|
sock_owned_by_me((const struct sock *)msk);
|
|
|
|
|
2020-06-30 04:26:20 +08:00
|
|
|
if (likely(!__mptcp_check_fallback(msk)))
|
2020-01-22 08:56:18 +08:00
|
|
|
return NULL;
|
|
|
|
|
2020-06-30 04:26:24 +08:00
|
|
|
return msk->first;
|
2020-01-22 08:56:18 +08:00
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:23 +08:00
|
|
|
static int __mptcp_socket_create(struct mptcp_sock *msk)
|
2020-01-22 08:56:17 +08:00
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
struct socket *ssock;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = mptcp_subflow_create_socket(sk, &ssock);
|
|
|
|
if (err)
|
2020-06-30 04:26:23 +08:00
|
|
|
return err;
|
2020-01-22 08:56:17 +08:00
|
|
|
|
mptcp: cope with later TCP fallback
With MPTCP v1, passive connections can fallback to TCP after the
subflow becomes established:
syn + MP_CAPABLE ->
<- syn, ack + MP_CAPABLE
ack, seq = 3 ->
// OoO packet is accepted because in-sequence
// passive socket is created, is in ESTABLISHED
// status and tentatively as MP_CAPABLE
ack, seq = 2 ->
// no MP_CAPABLE opt, subflow should fallback to TCP
We can't use the 'subflow' socket fallback, as we don't have
it available for passive connection.
Instead, when the fallback is detected, replace the mptcp
socket with the underlying TCP subflow. Beyond covering
the above scenario, it makes a TCP fallback socket as efficient
as plain TCP ones.
Co-developed-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-22 08:56:33 +08:00
|
|
|
msk->first = ssock->sk;
|
2020-01-22 08:56:17 +08:00
|
|
|
msk->subflow = ssock;
|
|
|
|
subflow = mptcp_subflow_ctx(ssock->sk);
|
2020-01-22 08:56:18 +08:00
|
|
|
list_add(&subflow->node, &msk->conn_list);
|
2020-01-22 08:56:17 +08:00
|
|
|
subflow->request_mptcp = 1;
|
|
|
|
|
2020-06-30 04:26:20 +08:00
|
|
|
/* accept() will wait on first subflow sk_wq, and we always wakes up
|
|
|
|
* via msk->sk_socket
|
|
|
|
*/
|
|
|
|
RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq);
|
|
|
|
|
2020-06-30 04:26:23 +08:00
|
|
|
return 0;
|
2020-01-22 08:56:17 +08:00
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
static void mptcp_drop(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
sk_drops_add(sk, skb);
|
|
|
|
__kfree_skb(skb);
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:11 +08:00
|
|
|
static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to,
|
|
|
|
struct sk_buff *from)
|
|
|
|
{
|
|
|
|
bool fragstolen;
|
|
|
|
int delta;
|
|
|
|
|
|
|
|
if (MPTCP_SKB_CB(from)->offset ||
|
|
|
|
!skb_try_coalesce(to, from, &fragstolen, &delta))
|
|
|
|
return false;
|
|
|
|
|
2020-09-14 16:01:14 +08:00
|
|
|
pr_debug("colesced seq %llx into %llx new len %d new end seq %llx",
|
|
|
|
MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq,
|
|
|
|
to->len, MPTCP_SKB_CB(from)->end_seq);
|
2020-09-14 16:01:12 +08:00
|
|
|
MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq;
|
2020-09-14 16:01:11 +08:00
|
|
|
kfree_skb_partial(from, fragstolen);
|
|
|
|
atomic_add(delta, &sk->sk_rmem_alloc);
|
|
|
|
sk_mem_charge(sk, delta);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to,
|
|
|
|
struct sk_buff *from)
|
|
|
|
{
|
|
|
|
if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return mptcp_try_coalesce((struct sock *)msk, to, from);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* "inspired" by tcp_data_queue_ofo(), main differences:
|
|
|
|
* - use mptcp seqs
|
|
|
|
* - don't cope with sacks
|
|
|
|
*/
|
|
|
|
static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
struct rb_node **p, *parent;
|
|
|
|
u64 seq, end_seq, max_seq;
|
|
|
|
struct sk_buff *skb1;
|
2020-09-17 09:12:33 +08:00
|
|
|
int space;
|
2020-09-14 16:01:12 +08:00
|
|
|
|
|
|
|
seq = MPTCP_SKB_CB(skb)->map_seq;
|
|
|
|
end_seq = MPTCP_SKB_CB(skb)->end_seq;
|
2020-09-17 09:12:33 +08:00
|
|
|
space = tcp_space(sk);
|
|
|
|
max_seq = space > 0 ? space + msk->ack_seq : msk->ack_seq;
|
2020-09-14 16:01:12 +08:00
|
|
|
|
2020-09-14 16:01:14 +08:00
|
|
|
pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq,
|
|
|
|
RB_EMPTY_ROOT(&msk->out_of_order_queue));
|
2020-09-14 16:01:12 +08:00
|
|
|
if (after64(seq, max_seq)) {
|
|
|
|
/* out of window */
|
|
|
|
mptcp_drop(sk, skb);
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
|
2020-09-14 16:01:12 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
p = &msk->out_of_order_queue.rb_node;
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE);
|
2020-09-14 16:01:12 +08:00
|
|
|
if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) {
|
|
|
|
rb_link_node(&skb->rbnode, NULL, p);
|
|
|
|
rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
|
|
|
|
msk->ooo_last_skb = skb;
|
|
|
|
goto end;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* with 2 subflows, adding at end of ooo queue is quite likely
|
|
|
|
* Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
|
|
|
|
*/
|
2020-09-14 16:01:14 +08:00
|
|
|
if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) {
|
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
|
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
|
2020-09-14 16:01:12 +08:00
|
|
|
return;
|
2020-09-14 16:01:14 +08:00
|
|
|
}
|
2020-09-14 16:01:12 +08:00
|
|
|
|
|
|
|
/* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
|
|
|
|
if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) {
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
|
2020-09-14 16:01:12 +08:00
|
|
|
parent = &msk->ooo_last_skb->rbnode;
|
|
|
|
p = &parent->rb_right;
|
|
|
|
goto insert;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Find place to insert this segment. Handle overlaps on the way. */
|
|
|
|
parent = NULL;
|
|
|
|
while (*p) {
|
|
|
|
parent = *p;
|
|
|
|
skb1 = rb_to_skb(parent);
|
|
|
|
if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
|
|
|
|
p = &parent->rb_left;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) {
|
|
|
|
if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) {
|
|
|
|
/* All the bits are present. Drop. */
|
|
|
|
mptcp_drop(sk, skb);
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
|
2020-09-14 16:01:12 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
|
|
|
|
/* partial overlap:
|
|
|
|
* | skb |
|
|
|
|
* | skb1 |
|
|
|
|
* continue traversing
|
|
|
|
*/
|
|
|
|
} else {
|
|
|
|
/* skb's seq == skb1's seq and skb covers skb1.
|
|
|
|
* Replace skb1 with skb.
|
|
|
|
*/
|
|
|
|
rb_replace_node(&skb1->rbnode, &skb->rbnode,
|
|
|
|
&msk->out_of_order_queue);
|
|
|
|
mptcp_drop(sk, skb1);
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
|
2020-09-14 16:01:12 +08:00
|
|
|
goto merge_right;
|
|
|
|
}
|
|
|
|
} else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) {
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
|
2020-09-14 16:01:12 +08:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
p = &parent->rb_right;
|
|
|
|
}
|
2020-09-14 16:01:14 +08:00
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
insert:
|
|
|
|
/* Insert segment into RB tree. */
|
|
|
|
rb_link_node(&skb->rbnode, parent, p);
|
|
|
|
rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
|
|
|
|
|
|
|
|
merge_right:
|
|
|
|
/* Remove other segments covered by skb. */
|
|
|
|
while ((skb1 = skb_rb_next(skb)) != NULL) {
|
|
|
|
if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq))
|
|
|
|
break;
|
|
|
|
rb_erase(&skb1->rbnode, &msk->out_of_order_queue);
|
|
|
|
mptcp_drop(sk, skb1);
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
|
2020-09-14 16:01:12 +08:00
|
|
|
}
|
|
|
|
/* If there is no skb after us, we are the last_skb ! */
|
|
|
|
if (!skb1)
|
|
|
|
msk->ooo_last_skb = skb;
|
|
|
|
|
|
|
|
end:
|
|
|
|
skb_condense(skb);
|
|
|
|
skb_set_owner_r(skb, sk);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,
|
|
|
|
struct sk_buff *skb, unsigned int offset,
|
|
|
|
size_t copy_len)
|
2020-02-26 17:14:48 +08:00
|
|
|
{
|
2020-09-14 16:01:12 +08:00
|
|
|
struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
|
2020-02-26 17:14:48 +08:00
|
|
|
struct sock *sk = (struct sock *)msk;
|
2020-05-26 05:41:13 +08:00
|
|
|
struct sk_buff *tail;
|
2020-02-26 17:14:48 +08:00
|
|
|
|
|
|
|
__skb_unlink(skb, &ssk->sk_receive_queue);
|
|
|
|
|
2020-05-26 05:41:13 +08:00
|
|
|
skb_ext_reset(skb);
|
|
|
|
skb_orphan(skb);
|
2020-09-14 16:01:12 +08:00
|
|
|
|
2020-10-27 22:59:14 +08:00
|
|
|
/* try to fetch required memory from subflow */
|
|
|
|
if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
|
|
|
|
if (ssk->sk_forward_alloc < skb->truesize)
|
|
|
|
goto drop;
|
|
|
|
__sk_mem_reclaim(ssk, skb->truesize);
|
|
|
|
if (!sk_rmem_schedule(sk, skb, skb->truesize))
|
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
/* the skb map_seq accounts for the skb offset:
|
|
|
|
* mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq
|
|
|
|
* value
|
|
|
|
*/
|
|
|
|
MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);
|
|
|
|
MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len;
|
2020-09-14 16:01:11 +08:00
|
|
|
MPTCP_SKB_CB(skb)->offset = offset;
|
2020-05-26 05:41:13 +08:00
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) {
|
|
|
|
/* in sequence */
|
2020-10-06 08:33:26 +08:00
|
|
|
WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
|
2020-09-14 16:01:12 +08:00
|
|
|
tail = skb_peek_tail(&sk->sk_receive_queue);
|
|
|
|
if (tail && mptcp_try_coalesce(sk, tail, skb))
|
|
|
|
return true;
|
2020-05-26 05:41:13 +08:00
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
skb_set_owner_r(skb, sk);
|
|
|
|
__skb_queue_tail(&sk->sk_receive_queue, skb);
|
|
|
|
return true;
|
|
|
|
} else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) {
|
|
|
|
mptcp_data_queue_ofo(msk, skb);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* old data, keep it simple and drop the whole pkt, sender
|
|
|
|
* will retransmit as needed, if needed.
|
|
|
|
*/
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
|
2020-10-27 22:59:14 +08:00
|
|
|
drop:
|
2020-09-14 16:01:12 +08:00
|
|
|
mptcp_drop(sk, skb);
|
|
|
|
return false;
|
2020-02-26 17:14:48 +08:00
|
|
|
}
|
|
|
|
|
2020-07-29 06:12:05 +08:00
|
|
|
static void mptcp_stop_timer(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
|
|
|
|
sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
|
|
|
|
mptcp_sk(sk)->timer_ival = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mptcp_check_data_fin_ack(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
if (__mptcp_check_fallback(msk))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Look for an acknowledged DATA_FIN */
|
|
|
|
if (((1 << sk->sk_state) &
|
|
|
|
(TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) &&
|
|
|
|
msk->write_seq == atomic64_read(&msk->snd_una)) {
|
|
|
|
mptcp_stop_timer(sk);
|
|
|
|
|
|
|
|
WRITE_ONCE(msk->snd_data_fin_enable, 0);
|
|
|
|
|
|
|
|
switch (sk->sk_state) {
|
|
|
|
case TCP_FIN_WAIT1:
|
|
|
|
inet_sk_state_store(sk, TCP_FIN_WAIT2);
|
|
|
|
sk->sk_state_change(sk);
|
|
|
|
break;
|
|
|
|
case TCP_CLOSING:
|
|
|
|
case TCP_LAST_ACK:
|
|
|
|
inet_sk_state_store(sk, TCP_CLOSE);
|
|
|
|
sk->sk_state_change(sk);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sk->sk_shutdown == SHUTDOWN_MASK ||
|
|
|
|
sk->sk_state == TCP_CLOSE)
|
|
|
|
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
|
|
|
|
else
|
|
|
|
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-29 06:12:03 +08:00
|
|
|
static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
if (READ_ONCE(msk->rcv_data_fin) &&
|
|
|
|
((1 << sk->sk_state) &
|
|
|
|
(TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) {
|
|
|
|
u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq);
|
|
|
|
|
|
|
|
if (msk->ack_seq == rcv_data_fin_seq) {
|
|
|
|
if (seq)
|
|
|
|
*seq = rcv_data_fin_seq;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk)
|
|
|
|
{
|
|
|
|
long tout = ssk && inet_csk(ssk)->icsk_pending ?
|
|
|
|
inet_csk(ssk)->icsk_timeout - jiffies : 0;
|
|
|
|
|
|
|
|
if (tout <= 0)
|
|
|
|
tout = mptcp_sk(sk)->timer_ival;
|
|
|
|
mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mptcp_check_data_fin(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
u64 rcv_data_fin_seq;
|
|
|
|
|
|
|
|
if (__mptcp_check_fallback(msk) || !msk->first)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* Need to ack a DATA_FIN received from a peer while this side
|
|
|
|
* of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2.
|
|
|
|
* msk->rcv_data_fin was set when parsing the incoming options
|
|
|
|
* at the subflow level and the msk lock was not held, so this
|
|
|
|
* is the first opportunity to act on the DATA_FIN and change
|
|
|
|
* the msk state.
|
|
|
|
*
|
|
|
|
* If we are caught up to the sequence number of the incoming
|
|
|
|
* DATA_FIN, send the DATA_ACK now and do state transition. If
|
|
|
|
* not caught up, do nothing and let the recv code send DATA_ACK
|
|
|
|
* when catching up.
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
|
2020-09-30 06:08:19 +08:00
|
|
|
WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1);
|
2020-07-29 06:12:03 +08:00
|
|
|
WRITE_ONCE(msk->rcv_data_fin, 0);
|
|
|
|
|
|
|
|
sk->sk_shutdown |= RCV_SHUTDOWN;
|
2020-07-29 06:12:05 +08:00
|
|
|
smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
|
|
|
|
set_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-07-29 06:12:03 +08:00
|
|
|
|
|
|
|
switch (sk->sk_state) {
|
|
|
|
case TCP_ESTABLISHED:
|
|
|
|
inet_sk_state_store(sk, TCP_CLOSE_WAIT);
|
|
|
|
break;
|
|
|
|
case TCP_FIN_WAIT1:
|
|
|
|
inet_sk_state_store(sk, TCP_CLOSING);
|
|
|
|
break;
|
|
|
|
case TCP_FIN_WAIT2:
|
|
|
|
inet_sk_state_store(sk, TCP_CLOSE);
|
|
|
|
// @@ Close subflows now?
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
/* Other states not expected */
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
mptcp_set_timeout(sk, NULL);
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
|
|
|
lock_sock(ssk);
|
|
|
|
tcp_send_ack(ssk);
|
|
|
|
release_sock(ssk);
|
|
|
|
}
|
|
|
|
|
|
|
|
sk->sk_state_change(sk);
|
|
|
|
|
|
|
|
if (sk->sk_shutdown == SHUTDOWN_MASK ||
|
|
|
|
sk->sk_state == TCP_CLOSE)
|
|
|
|
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
|
|
|
|
else
|
|
|
|
sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
|
|
|
|
struct sock *ssk,
|
|
|
|
unsigned int *bytes)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
|
2020-02-26 17:14:49 +08:00
|
|
|
struct sock *sk = (struct sock *)msk;
|
2020-02-26 17:14:48 +08:00
|
|
|
unsigned int moved = 0;
|
|
|
|
bool more_data_avail;
|
|
|
|
struct tcp_sock *tp;
|
2020-10-05 18:36:44 +08:00
|
|
|
u32 old_copied_seq;
|
2020-02-26 17:14:48 +08:00
|
|
|
bool done = false;
|
2020-02-26 17:14:49 +08:00
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
pr_debug("msk=%p ssk=%p", msk, ssk);
|
2020-02-26 17:14:48 +08:00
|
|
|
tp = tcp_sk(ssk);
|
2020-10-05 18:36:44 +08:00
|
|
|
old_copied_seq = tp->copied_seq;
|
2020-02-26 17:14:48 +08:00
|
|
|
do {
|
|
|
|
u32 map_remaining, offset;
|
|
|
|
u32 seq = tp->copied_seq;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
bool fin;
|
|
|
|
|
|
|
|
/* try to move as much data as available */
|
|
|
|
map_remaining = subflow->map_data_len -
|
|
|
|
mptcp_subflow_get_map_offset(subflow);
|
|
|
|
|
|
|
|
skb = skb_peek(&ssk->sk_receive_queue);
|
2020-10-06 14:27:34 +08:00
|
|
|
if (!skb) {
|
|
|
|
/* if no data is found, a racing workqueue/recvmsg
|
|
|
|
* already processed the new data, stop here or we
|
|
|
|
* can enter an infinite loop
|
|
|
|
*/
|
|
|
|
if (!moved)
|
|
|
|
done = true;
|
2020-02-26 17:14:48 +08:00
|
|
|
break;
|
2020-10-06 14:27:34 +08:00
|
|
|
}
|
2020-02-26 17:14:48 +08:00
|
|
|
|
2020-06-30 04:26:20 +08:00
|
|
|
if (__mptcp_check_fallback(msk)) {
|
|
|
|
/* if we are running under the workqueue, TCP could have
|
|
|
|
* collapsed skbs between dummy map creation and now
|
|
|
|
* be sure to adjust the size
|
|
|
|
*/
|
|
|
|
map_remaining = skb->len;
|
|
|
|
subflow->map_data_len = skb->len;
|
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
offset = seq - TCP_SKB_CB(skb)->seq;
|
|
|
|
fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
|
|
|
|
if (fin) {
|
|
|
|
done = true;
|
|
|
|
seq++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (offset < skb->len) {
|
|
|
|
size_t len = skb->len - offset;
|
|
|
|
|
|
|
|
if (tp->urg_data)
|
|
|
|
done = true;
|
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
if (__mptcp_move_skb(msk, ssk, skb, offset, len))
|
|
|
|
moved += len;
|
2020-02-26 17:14:48 +08:00
|
|
|
seq += len;
|
|
|
|
|
|
|
|
if (WARN_ON_ONCE(map_remaining < len))
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
WARN_ON_ONCE(!fin);
|
|
|
|
sk_eat_skb(ssk, skb);
|
|
|
|
done = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
WRITE_ONCE(tp->copied_seq, seq);
|
|
|
|
more_data_avail = mptcp_subflow_data_available(ssk);
|
2020-02-26 17:14:49 +08:00
|
|
|
|
|
|
|
if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) {
|
|
|
|
done = true;
|
|
|
|
break;
|
|
|
|
}
|
2020-02-26 17:14:48 +08:00
|
|
|
} while (more_data_avail);
|
|
|
|
|
2020-09-14 16:01:09 +08:00
|
|
|
*bytes += moved;
|
2020-10-05 18:36:44 +08:00
|
|
|
if (tp->copied_seq != old_copied_seq)
|
|
|
|
tcp_cleanup_rbuf(ssk, 1);
|
2020-02-26 17:14:48 +08:00
|
|
|
|
|
|
|
return done;
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
static bool mptcp_ofo_queue(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
struct sk_buff *skb, *tail;
|
|
|
|
bool moved = false;
|
|
|
|
struct rb_node *p;
|
|
|
|
u64 end_seq;
|
|
|
|
|
|
|
|
p = rb_first(&msk->out_of_order_queue);
|
2020-09-14 16:01:14 +08:00
|
|
|
pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue));
|
2020-09-14 16:01:12 +08:00
|
|
|
while (p) {
|
|
|
|
skb = rb_to_skb(p);
|
|
|
|
if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq))
|
|
|
|
break;
|
|
|
|
|
|
|
|
p = rb_next(p);
|
|
|
|
rb_erase(&skb->rbnode, &msk->out_of_order_queue);
|
|
|
|
|
|
|
|
if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq,
|
|
|
|
msk->ack_seq))) {
|
|
|
|
mptcp_drop(sk, skb);
|
2020-09-14 16:01:14 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
|
2020-09-14 16:01:12 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
end_seq = MPTCP_SKB_CB(skb)->end_seq;
|
|
|
|
tail = skb_peek_tail(&sk->sk_receive_queue);
|
|
|
|
if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) {
|
|
|
|
int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq;
|
|
|
|
|
|
|
|
/* skip overlapping data, if any */
|
2020-09-14 16:01:14 +08:00
|
|
|
pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d",
|
|
|
|
MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq,
|
|
|
|
delta);
|
2020-09-14 16:01:12 +08:00
|
|
|
MPTCP_SKB_CB(skb)->offset += delta;
|
|
|
|
__skb_queue_tail(&sk->sk_receive_queue, skb);
|
|
|
|
}
|
|
|
|
msk->ack_seq = end_seq;
|
|
|
|
moved = true;
|
|
|
|
}
|
|
|
|
return moved;
|
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:51 +08:00
|
|
|
/* In most cases we will be able to lock the mptcp socket. If its already
|
|
|
|
* owned, we need to defer to the work queue to avoid ABBA deadlock.
|
|
|
|
*/
|
|
|
|
static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)
|
|
|
|
{
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
unsigned int moved = 0;
|
|
|
|
|
|
|
|
if (READ_ONCE(sk->sk_lock.owned))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock)))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* must re-check after taking the lock */
|
2020-09-14 16:01:12 +08:00
|
|
|
if (!READ_ONCE(sk->sk_lock.owned)) {
|
2020-02-26 17:14:51 +08:00
|
|
|
__mptcp_move_skbs_from_subflow(msk, ssk, &moved);
|
2020-09-14 16:01:12 +08:00
|
|
|
mptcp_ofo_queue(msk);
|
|
|
|
|
|
|
|
/* If the moves have caught up with the DATA_FIN sequence number
|
|
|
|
* it's time to ack the DATA_FIN and change socket state, but
|
|
|
|
* this is not a good place to change state. Let the workqueue
|
|
|
|
* do it.
|
|
|
|
*/
|
|
|
|
if (mptcp_pending_data_fin(sk, NULL) &&
|
|
|
|
schedule_work(&msk->work))
|
|
|
|
sock_hold(sk);
|
|
|
|
}
|
2020-02-26 17:14:51 +08:00
|
|
|
|
|
|
|
spin_unlock_bh(&sk->sk_lock.slock);
|
|
|
|
|
|
|
|
return moved > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void mptcp_data_ready(struct sock *sk, struct sock *ssk)
|
2020-02-26 17:14:46 +08:00
|
|
|
{
|
2020-09-14 16:01:09 +08:00
|
|
|
struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
|
2020-02-26 17:14:46 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
2020-09-14 16:01:09 +08:00
|
|
|
bool wake;
|
2020-02-26 17:14:46 +08:00
|
|
|
|
2020-09-14 16:01:09 +08:00
|
|
|
/* move_skbs_to_msk below can legitly clear the data_avail flag,
|
|
|
|
* but we will need later to properly woke the reader, cache its
|
|
|
|
* value
|
|
|
|
*/
|
|
|
|
wake = subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL;
|
|
|
|
if (wake)
|
|
|
|
set_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-02-26 17:14:48 +08:00
|
|
|
|
2020-02-26 17:14:51 +08:00
|
|
|
if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) &&
|
|
|
|
move_skbs_to_msk(msk, ssk))
|
|
|
|
goto wake;
|
|
|
|
|
2020-02-26 17:14:49 +08:00
|
|
|
/* don't schedule if mptcp sk is (still) over limit */
|
|
|
|
if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf))
|
|
|
|
goto wake;
|
|
|
|
|
2020-02-26 17:14:52 +08:00
|
|
|
/* mptcp socket is owned, release_cb should retry */
|
|
|
|
if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED,
|
|
|
|
&sk->sk_tsq_flags)) {
|
|
|
|
sock_hold(sk);
|
2020-02-26 17:14:48 +08:00
|
|
|
|
2020-02-26 17:14:52 +08:00
|
|
|
/* need to try again, its possible release_cb() has already
|
|
|
|
* been called after the test_and_set_bit() above.
|
|
|
|
*/
|
|
|
|
move_skbs_to_msk(msk, ssk);
|
|
|
|
}
|
2020-02-26 17:14:49 +08:00
|
|
|
wake:
|
2020-09-14 16:01:09 +08:00
|
|
|
if (wake)
|
|
|
|
sk->sk_data_ready(sk);
|
2020-02-26 17:14:46 +08:00
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:40 +08:00
|
|
|
static void __mptcp_flush_join_list(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
if (likely(list_empty(&msk->join_list)))
|
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock_bh(&msk->join_list_lock);
|
|
|
|
list_splice_tail_init(&msk->join_list, &msk->conn_list);
|
|
|
|
spin_unlock_bh(&msk->join_list_lock);
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
static bool mptcp_timer_pending(struct sock *sk)
|
|
|
|
{
|
|
|
|
return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mptcp_reset_timer(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
unsigned long tout;
|
|
|
|
|
|
|
|
/* should never be called with mptcp level timer cleared */
|
|
|
|
tout = READ_ONCE(mptcp_sk(sk)->timer_ival);
|
|
|
|
if (WARN_ON_ONCE(!tout))
|
|
|
|
tout = TCP_RTO_MIN;
|
|
|
|
sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);
|
|
|
|
}
|
|
|
|
|
|
|
|
void mptcp_data_acked(struct sock *sk)
|
|
|
|
{
|
|
|
|
mptcp_reset_timer(sk);
|
2020-03-28 05:48:48 +08:00
|
|
|
|
2020-09-14 16:01:07 +08:00
|
|
|
if ((!test_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags) ||
|
2020-07-29 06:12:06 +08:00
|
|
|
(inet_sk_state_load(sk) != TCP_ESTABLISHED)) &&
|
2020-03-28 05:48:48 +08:00
|
|
|
schedule_work(&mptcp_sk(sk)->work))
|
|
|
|
sock_hold(sk);
|
2020-03-28 05:48:44 +08:00
|
|
|
}
|
|
|
|
|
2020-04-02 19:44:52 +08:00
|
|
|
void mptcp_subflow_eof(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) &&
|
|
|
|
schedule_work(&msk->work))
|
|
|
|
sock_hold(sk);
|
|
|
|
}
|
|
|
|
|
2020-06-10 16:47:41 +08:00
|
|
|
static void mptcp_check_for_eof(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
int receivers = 0;
|
|
|
|
|
|
|
|
mptcp_for_each_subflow(msk, subflow)
|
|
|
|
receivers += !subflow->rx_eof;
|
|
|
|
|
|
|
|
if (!receivers && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
|
|
|
|
/* hopefully temporary hack: propagate shutdown status
|
|
|
|
* to msk, when all subflows agree on it
|
|
|
|
*/
|
|
|
|
sk->sk_shutdown |= RCV_SHUTDOWN;
|
|
|
|
|
|
|
|
smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
|
|
|
|
set_bit(MPTCP_DATA_READY, &msk->flags);
|
|
|
|
sk->sk_data_ready(sk);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
static bool mptcp_ext_cache_refill(struct mptcp_sock *msk)
|
|
|
|
{
|
2020-05-16 16:46:23 +08:00
|
|
|
const struct sock *sk = (const struct sock *)msk;
|
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
if (!msk->cached_ext)
|
2020-05-16 16:46:23 +08:00
|
|
|
msk->cached_ext = __skb_ext_alloc(sk->sk_allocation);
|
2020-01-22 08:56:23 +08:00
|
|
|
|
|
|
|
return !!msk->cached_ext;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:26 +08:00
|
|
|
static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
|
|
|
|
sock_owned_by_me(sk);
|
|
|
|
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
if (subflow->data_avail)
|
|
|
|
return mptcp_subflow_tcp_sock(subflow);
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:47 +08:00
|
|
|
static bool mptcp_skb_can_collapse_to(u64 write_seq,
|
|
|
|
const struct sk_buff *skb,
|
|
|
|
const struct mptcp_ext *mpext)
|
2020-01-22 08:56:27 +08:00
|
|
|
{
|
|
|
|
if (!tcp_skb_can_collapse_to(skb))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* can collapse only if MPTCP level sequence is in order */
|
2020-03-28 05:48:47 +08:00
|
|
|
return mpext && mpext->data_seq + mpext->data_len == write_seq;
|
2020-01-22 08:56:27 +08:00
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk,
|
|
|
|
const struct page_frag *pfrag,
|
|
|
|
const struct mptcp_data_frag *df)
|
|
|
|
{
|
|
|
|
return df && pfrag->page == df->page &&
|
|
|
|
df->data_seq + df->data_len == msk->write_seq;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:45 +08:00
|
|
|
static void dfrag_uncharge(struct sock *sk, int len)
|
|
|
|
{
|
|
|
|
sk_mem_uncharge(sk, len);
|
2020-03-28 05:48:46 +08:00
|
|
|
sk_wmem_queued_add(sk, -len);
|
2020-03-28 05:48:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)
|
2020-03-28 05:48:43 +08:00
|
|
|
{
|
2020-03-28 05:48:45 +08:00
|
|
|
int len = dfrag->data_len + dfrag->overhead;
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
list_del(&dfrag->list);
|
2020-03-28 05:48:45 +08:00
|
|
|
dfrag_uncharge(sk, len);
|
2020-03-28 05:48:43 +08:00
|
|
|
put_page(dfrag->page);
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:07 +08:00
|
|
|
static bool mptcp_is_writeable(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
|
|
|
|
if (!sk_stream_is_writeable((struct sock *)msk))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
if (sk_stream_is_writeable(subflow->tcp_sock))
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
static void mptcp_clean_una(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
struct mptcp_data_frag *dtmp, *dfrag;
|
2020-03-28 05:48:45 +08:00
|
|
|
bool cleaned = false;
|
2020-06-30 04:26:20 +08:00
|
|
|
u64 snd_una;
|
|
|
|
|
|
|
|
/* on fallback we just need to ignore snd_una, as this is really
|
|
|
|
* plain TCP
|
|
|
|
*/
|
|
|
|
if (__mptcp_check_fallback(msk))
|
|
|
|
atomic64_set(&msk->snd_una, msk->write_seq);
|
|
|
|
snd_una = atomic64_read(&msk->snd_una);
|
2020-03-28 05:48:43 +08:00
|
|
|
|
|
|
|
list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {
|
|
|
|
if (after64(dfrag->data_seq + dfrag->data_len, snd_una))
|
|
|
|
break;
|
|
|
|
|
2020-03-28 05:48:45 +08:00
|
|
|
dfrag_clear(sk, dfrag);
|
|
|
|
cleaned = true;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:46 +08:00
|
|
|
dfrag = mptcp_rtx_head(sk);
|
|
|
|
if (dfrag && after64(snd_una, dfrag->data_seq)) {
|
2020-07-23 19:02:30 +08:00
|
|
|
u64 delta = snd_una - dfrag->data_seq;
|
|
|
|
|
|
|
|
if (WARN_ON_ONCE(delta > dfrag->data_len))
|
|
|
|
goto out;
|
2020-03-28 05:48:46 +08:00
|
|
|
|
|
|
|
dfrag->data_seq += delta;
|
2020-07-23 19:02:30 +08:00
|
|
|
dfrag->offset += delta;
|
2020-03-28 05:48:46 +08:00
|
|
|
dfrag->data_len -= delta;
|
|
|
|
|
|
|
|
dfrag_uncharge(sk, delta);
|
|
|
|
cleaned = true;
|
|
|
|
}
|
|
|
|
|
2020-07-23 19:02:30 +08:00
|
|
|
out:
|
2020-03-28 05:48:45 +08:00
|
|
|
if (cleaned) {
|
|
|
|
sk_mem_reclaim_partial(sk);
|
2020-03-28 05:48:46 +08:00
|
|
|
|
|
|
|
/* Only wake up writers if a subflow is ready */
|
2020-09-14 16:01:07 +08:00
|
|
|
if (mptcp_is_writeable(msk)) {
|
|
|
|
set_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags);
|
|
|
|
smp_mb__after_atomic();
|
|
|
|
|
|
|
|
/* set SEND_SPACE before sk_stream_write_space clears
|
|
|
|
* NOSPACE
|
|
|
|
*/
|
2020-03-28 05:48:46 +08:00
|
|
|
sk_stream_write_space(sk);
|
2020-09-14 16:01:07 +08:00
|
|
|
}
|
2020-03-28 05:48:43 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* ensure we get enough memory for the frag hdr, beyond some minimal amount of
|
|
|
|
* data
|
|
|
|
*/
|
|
|
|
static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
|
|
|
|
{
|
|
|
|
if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag),
|
|
|
|
pfrag, sk->sk_allocation)))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
sk->sk_prot->enter_memory_pressure(sk);
|
|
|
|
sk_stream_moderate_sndbuf(sk);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct mptcp_data_frag *
|
|
|
|
mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag,
|
|
|
|
int orig_offset)
|
|
|
|
{
|
|
|
|
int offset = ALIGN(orig_offset, sizeof(long));
|
|
|
|
struct mptcp_data_frag *dfrag;
|
|
|
|
|
|
|
|
dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);
|
|
|
|
dfrag->data_len = 0;
|
|
|
|
dfrag->data_seq = msk->write_seq;
|
|
|
|
dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);
|
|
|
|
dfrag->offset = offset + sizeof(struct mptcp_data_frag);
|
|
|
|
dfrag->page = pfrag->page;
|
|
|
|
|
|
|
|
return dfrag;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
|
2020-03-28 05:48:47 +08:00
|
|
|
struct msghdr *msg, struct mptcp_data_frag *dfrag,
|
|
|
|
long *timeo, int *pmss_now,
|
2020-01-22 08:56:27 +08:00
|
|
|
int *ps_goal)
|
2020-01-22 08:56:23 +08:00
|
|
|
{
|
2020-03-28 05:48:43 +08:00
|
|
|
int mss_now, avail_size, size_goal, offset, ret, frag_truesize = 0;
|
|
|
|
bool dfrag_collapsed, can_collapse = false;
|
2020-01-22 08:56:23 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
struct mptcp_ext *mpext = NULL;
|
2020-03-28 05:48:47 +08:00
|
|
|
bool retransmission = !!dfrag;
|
2020-01-22 08:56:27 +08:00
|
|
|
struct sk_buff *skb, *tail;
|
2020-01-22 08:56:23 +08:00
|
|
|
struct page_frag *pfrag;
|
2020-03-28 05:48:47 +08:00
|
|
|
struct page *page;
|
|
|
|
u64 *write_seq;
|
2020-01-22 08:56:23 +08:00
|
|
|
size_t psize;
|
|
|
|
|
|
|
|
/* use the mptcp page cache so that we can easily move the data
|
|
|
|
* from one substream to another, but do per subflow memory accounting
|
2020-03-28 05:48:47 +08:00
|
|
|
* Note: pfrag is used only !retransmission, but the compiler if
|
|
|
|
* fooled into a warning if we don't init here
|
2020-01-22 08:56:23 +08:00
|
|
|
*/
|
|
|
|
pfrag = sk_page_frag(sk);
|
2020-03-28 05:48:47 +08:00
|
|
|
if (!retransmission) {
|
|
|
|
write_seq = &msk->write_seq;
|
|
|
|
page = pfrag->page;
|
|
|
|
} else {
|
|
|
|
write_seq = &dfrag->data_seq;
|
|
|
|
page = dfrag->page;
|
|
|
|
}
|
2020-01-22 08:56:23 +08:00
|
|
|
|
|
|
|
/* compute copy limit */
|
|
|
|
mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags);
|
2020-01-22 08:56:27 +08:00
|
|
|
*pmss_now = mss_now;
|
|
|
|
*ps_goal = size_goal;
|
|
|
|
avail_size = size_goal;
|
|
|
|
skb = tcp_write_queue_tail(ssk);
|
|
|
|
if (skb) {
|
|
|
|
mpext = skb_ext_find(skb, SKB_EXT_MPTCP);
|
|
|
|
|
|
|
|
/* Limit the write to the size available in the
|
|
|
|
* current skb, if any, so that we create at most a new skb.
|
|
|
|
* Explicitly tells TCP internals to avoid collapsing on later
|
|
|
|
* queue management operation, to avoid breaking the ext <->
|
|
|
|
* SSN association set here
|
|
|
|
*/
|
|
|
|
can_collapse = (size_goal - skb->len > 0) &&
|
2020-03-28 05:48:47 +08:00
|
|
|
mptcp_skb_can_collapse_to(*write_seq, skb, mpext);
|
2020-01-22 08:56:27 +08:00
|
|
|
if (!can_collapse)
|
|
|
|
TCP_SKB_CB(skb)->eor = 1;
|
|
|
|
else
|
|
|
|
avail_size = size_goal - skb->len;
|
|
|
|
}
|
2020-03-28 05:48:43 +08:00
|
|
|
|
2020-03-28 05:48:47 +08:00
|
|
|
if (!retransmission) {
|
|
|
|
/* reuse tail pfrag, if possible, or carve a new one from the
|
|
|
|
* page allocator
|
|
|
|
*/
|
|
|
|
dfrag = mptcp_rtx_tail(sk);
|
|
|
|
offset = pfrag->offset;
|
|
|
|
dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);
|
|
|
|
if (!dfrag_collapsed) {
|
|
|
|
dfrag = mptcp_carve_data_frag(msk, pfrag, offset);
|
|
|
|
offset = dfrag->offset;
|
|
|
|
frag_truesize = dfrag->overhead;
|
|
|
|
}
|
|
|
|
psize = min_t(size_t, pfrag->size - offset, avail_size);
|
|
|
|
|
|
|
|
/* Copy to page */
|
|
|
|
pr_debug("left=%zu", msg_data_left(msg));
|
|
|
|
psize = copy_page_from_iter(pfrag->page, offset,
|
|
|
|
min_t(size_t, msg_data_left(msg),
|
|
|
|
psize),
|
|
|
|
&msg->msg_iter);
|
|
|
|
pr_debug("left=%zu", msg_data_left(msg));
|
|
|
|
if (!psize)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2020-08-14 21:56:34 +08:00
|
|
|
if (!sk_wmem_schedule(sk, psize + dfrag->overhead)) {
|
|
|
|
iov_iter_revert(&msg->msg_iter, psize);
|
2020-03-28 05:48:47 +08:00
|
|
|
return -ENOMEM;
|
2020-08-14 21:56:34 +08:00
|
|
|
}
|
2020-03-28 05:48:47 +08:00
|
|
|
} else {
|
2020-03-28 05:48:43 +08:00
|
|
|
offset = dfrag->offset;
|
2020-03-28 05:48:47 +08:00
|
|
|
psize = min_t(size_t, dfrag->data_len, avail_size);
|
2020-03-28 05:48:43 +08:00
|
|
|
}
|
2020-03-28 05:48:45 +08:00
|
|
|
|
2020-01-22 08:56:27 +08:00
|
|
|
/* tell the TCP stack to delay the push so that we can safely
|
|
|
|
* access the skb after the sendpages call
|
2020-01-22 08:56:23 +08:00
|
|
|
*/
|
2020-03-28 05:48:47 +08:00
|
|
|
ret = do_tcp_sendpages(ssk, page, offset, psize,
|
2020-05-16 16:46:19 +08:00
|
|
|
msg->msg_flags | MSG_SENDPAGE_NOTLAST | MSG_DONTWAIT);
|
2020-08-14 21:56:34 +08:00
|
|
|
if (ret <= 0) {
|
2020-08-17 05:14:20 +08:00
|
|
|
if (!retransmission)
|
|
|
|
iov_iter_revert(&msg->msg_iter, psize);
|
2020-01-22 08:56:23 +08:00
|
|
|
return ret;
|
2020-08-14 21:56:34 +08:00
|
|
|
}
|
2020-03-28 05:48:43 +08:00
|
|
|
|
|
|
|
frag_truesize += ret;
|
2020-03-28 05:48:47 +08:00
|
|
|
if (!retransmission) {
|
|
|
|
if (unlikely(ret < psize))
|
|
|
|
iov_iter_revert(&msg->msg_iter, psize - ret);
|
2020-01-22 08:56:23 +08:00
|
|
|
|
2020-03-28 05:48:47 +08:00
|
|
|
/* send successful, keep track of sent data for mptcp-level
|
|
|
|
* retransmission
|
|
|
|
*/
|
|
|
|
dfrag->data_len += ret;
|
|
|
|
if (!dfrag_collapsed) {
|
|
|
|
get_page(dfrag->page);
|
|
|
|
list_add_tail(&dfrag->list, &msk->rtx_queue);
|
|
|
|
sk_wmem_queued_add(sk, frag_truesize);
|
|
|
|
} else {
|
|
|
|
sk_wmem_queued_add(sk, ret);
|
|
|
|
}
|
2020-03-28 05:48:43 +08:00
|
|
|
|
2020-03-28 05:48:47 +08:00
|
|
|
/* charge data on mptcp rtx queue to the master socket
|
|
|
|
* Note: we charge such data both to sk and ssk
|
|
|
|
*/
|
|
|
|
sk->sk_forward_alloc -= frag_truesize;
|
|
|
|
}
|
2020-03-28 05:48:45 +08:00
|
|
|
|
2020-01-22 08:56:27 +08:00
|
|
|
/* if the tail skb extension is still the cached one, collapsing
|
|
|
|
* really happened. Note: we can't check for 'same skb' as the sk_buff
|
|
|
|
* hdr on tail can be transmitted, freed and re-allocated by the
|
|
|
|
* do_tcp_sendpages() call
|
|
|
|
*/
|
|
|
|
tail = tcp_write_queue_tail(ssk);
|
|
|
|
if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) {
|
|
|
|
WARN_ON_ONCE(!can_collapse);
|
|
|
|
mpext->data_len += ret;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
skb = tcp_write_queue_tail(ssk);
|
|
|
|
mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext);
|
|
|
|
msk->cached_ext = NULL;
|
|
|
|
|
|
|
|
memset(mpext, 0, sizeof(*mpext));
|
2020-03-28 05:48:47 +08:00
|
|
|
mpext->data_seq = *write_seq;
|
2020-01-22 08:56:23 +08:00
|
|
|
mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;
|
|
|
|
mpext->data_len = ret;
|
|
|
|
mpext->use_map = 1;
|
|
|
|
mpext->dsn64 = 1;
|
|
|
|
|
|
|
|
pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d",
|
|
|
|
mpext->data_seq, mpext->subflow_seq, mpext->data_len,
|
|
|
|
mpext->dsn64);
|
|
|
|
|
2020-01-22 08:56:27 +08:00
|
|
|
out:
|
2020-03-28 05:48:47 +08:00
|
|
|
if (!retransmission)
|
|
|
|
pfrag->offset += frag_truesize;
|
2020-07-29 06:12:10 +08:00
|
|
|
WRITE_ONCE(*write_seq, *write_seq + ret);
|
2020-01-22 08:56:23 +08:00
|
|
|
mptcp_subflow_ctx(ssk)->rel_write_seq += ret;
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:07 +08:00
|
|
|
static void mptcp_nospace(struct mptcp_sock *msk)
|
2020-05-16 16:46:17 +08:00
|
|
|
{
|
2020-09-14 16:01:07 +08:00
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
|
2020-05-16 16:46:17 +08:00
|
|
|
clear_bit(MPTCP_SEND_SPACE, &msk->flags);
|
|
|
|
smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */
|
|
|
|
|
2020-09-14 16:01:07 +08:00
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
struct socket *sock = READ_ONCE(ssk->sk_socket);
|
|
|
|
|
|
|
|
/* enables ssk->write_space() callbacks */
|
|
|
|
if (sock)
|
|
|
|
set_bit(SOCK_NOSPACE, &sock->flags);
|
|
|
|
}
|
2020-05-16 16:46:17 +08:00
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
static bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
|
|
|
|
{
|
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
|
|
|
/* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */
|
|
|
|
if (subflow->request_join && !subflow->fully_established)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* only send if our side has not closed yet */
|
|
|
|
return ((1 << ssk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT));
|
|
|
|
}
|
|
|
|
|
|
|
|
#define MPTCP_SEND_BURST_SIZE ((1 << 16) - \
|
|
|
|
sizeof(struct tcphdr) - \
|
|
|
|
MAX_TCP_OPTION_SPACE - \
|
|
|
|
sizeof(struct ipv6hdr) - \
|
|
|
|
sizeof(struct frag_hdr))
|
|
|
|
|
|
|
|
struct subflow_send_info {
|
|
|
|
struct sock *ssk;
|
|
|
|
u64 ratio;
|
|
|
|
};
|
|
|
|
|
2020-09-14 16:01:10 +08:00
|
|
|
static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk,
|
|
|
|
u32 *sndbuf)
|
2020-03-28 05:48:39 +08:00
|
|
|
{
|
2020-09-14 16:01:17 +08:00
|
|
|
struct subflow_send_info send_info[2];
|
2020-03-28 05:48:39 +08:00
|
|
|
struct mptcp_subflow_context *subflow;
|
2020-09-14 16:01:17 +08:00
|
|
|
int i, nr_active = 0;
|
|
|
|
struct sock *ssk;
|
|
|
|
u64 ratio;
|
|
|
|
u32 pace;
|
2020-03-28 05:48:39 +08:00
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
sock_owned_by_me((struct sock *)msk);
|
2020-03-28 05:48:39 +08:00
|
|
|
|
2020-09-14 16:01:10 +08:00
|
|
|
*sndbuf = 0;
|
2020-05-16 16:46:20 +08:00
|
|
|
if (!mptcp_ext_cache_refill(msk))
|
|
|
|
return NULL;
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
if (__mptcp_check_fallback(msk)) {
|
|
|
|
if (!msk->first)
|
2020-03-28 05:48:39 +08:00
|
|
|
return NULL;
|
2020-09-14 16:01:17 +08:00
|
|
|
*sndbuf = msk->first->sk_sndbuf;
|
|
|
|
return sk_stream_memory_free(msk->first) ? msk->first : NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* re-use last subflow, if the burst allow that */
|
|
|
|
if (msk->last_snd && msk->snd_burst > 0 &&
|
|
|
|
sk_stream_memory_free(msk->last_snd) &&
|
|
|
|
mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) {
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
*sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf);
|
2020-03-28 05:48:39 +08:00
|
|
|
}
|
2020-09-14 16:01:17 +08:00
|
|
|
return msk->last_snd;
|
|
|
|
}
|
2020-03-28 05:48:39 +08:00
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
/* pick the subflow with the lower wmem/wspace ratio */
|
|
|
|
for (i = 0; i < 2; ++i) {
|
|
|
|
send_info[i].ssk = NULL;
|
|
|
|
send_info[i].ratio = -1;
|
|
|
|
}
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
if (!mptcp_subflow_active(subflow))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
nr_active += !subflow->backup;
|
2020-09-14 16:01:10 +08:00
|
|
|
*sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf);
|
2020-09-14 16:01:17 +08:00
|
|
|
if (!sk_stream_memory_free(subflow->tcp_sock))
|
|
|
|
continue;
|
2020-03-28 05:48:39 +08:00
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
pace = READ_ONCE(ssk->sk_pacing_rate);
|
|
|
|
if (!pace)
|
2020-03-28 05:48:39 +08:00
|
|
|
continue;
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32,
|
|
|
|
pace);
|
|
|
|
if (ratio < send_info[subflow->backup].ratio) {
|
|
|
|
send_info[subflow->backup].ssk = ssk;
|
|
|
|
send_info[subflow->backup].ratio = ratio;
|
|
|
|
}
|
2020-03-28 05:48:39 +08:00
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
pr_debug("msk=%p nr_active=%d ssk=%p:%lld backup=%p:%lld",
|
|
|
|
msk, nr_active, send_info[0].ssk, send_info[0].ratio,
|
|
|
|
send_info[1].ssk, send_info[1].ratio);
|
|
|
|
|
|
|
|
/* pick the best backup if no other subflow is active */
|
|
|
|
if (!nr_active)
|
|
|
|
send_info[0].ssk = send_info[1].ssk;
|
|
|
|
|
|
|
|
if (send_info[0].ssk) {
|
|
|
|
msk->last_snd = send_info[0].ssk;
|
|
|
|
msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE,
|
|
|
|
sk_stream_wspace(msk->last_snd));
|
|
|
|
return msk->last_snd;
|
|
|
|
}
|
|
|
|
return NULL;
|
2020-03-28 05:48:39 +08:00
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:07 +08:00
|
|
|
static void ssk_check_wmem(struct mptcp_sock *msk)
|
2020-01-22 08:56:25 +08:00
|
|
|
{
|
2020-09-14 16:01:07 +08:00
|
|
|
if (unlikely(!mptcp_is_writeable(msk)))
|
|
|
|
mptcp_nospace(msk);
|
2020-01-22 08:56:25 +08:00
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
|
|
|
|
{
|
2020-01-22 08:56:27 +08:00
|
|
|
int mss_now = 0, size_goal = 0, ret = 0;
|
2020-01-22 08:56:15 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
2020-05-16 16:46:21 +08:00
|
|
|
struct page_frag *pfrag;
|
2020-01-22 08:56:23 +08:00
|
|
|
size_t copied = 0;
|
2020-01-22 08:56:18 +08:00
|
|
|
struct sock *ssk;
|
2020-09-14 16:01:10 +08:00
|
|
|
u32 sndbuf;
|
2020-05-16 16:46:19 +08:00
|
|
|
bool tx_ok;
|
2020-01-22 08:56:23 +08:00
|
|
|
long timeo;
|
2020-01-22 08:56:15 +08:00
|
|
|
|
|
|
|
if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
lock_sock(sk);
|
2020-02-29 07:47:39 +08:00
|
|
|
|
|
|
|
timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
|
|
|
|
|
|
|
|
if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
|
|
|
|
ret = sk_stream_wait_connect(sk, &timeo);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2020-05-16 16:46:21 +08:00
|
|
|
pfrag = sk_page_frag(sk);
|
2020-05-16 16:46:19 +08:00
|
|
|
restart:
|
2020-03-28 05:48:43 +08:00
|
|
|
mptcp_clean_una(sk);
|
|
|
|
|
2020-07-29 06:12:00 +08:00
|
|
|
if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) {
|
|
|
|
ret = -EPIPE;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:40 +08:00
|
|
|
__mptcp_flush_join_list(msk);
|
2020-09-14 16:01:10 +08:00
|
|
|
ssk = mptcp_subflow_get_send(msk, &sndbuf);
|
2020-05-16 16:46:21 +08:00
|
|
|
while (!sk_stream_memory_free(sk) ||
|
|
|
|
!ssk ||
|
|
|
|
!mptcp_page_frag_refill(ssk, pfrag)) {
|
2020-05-16 16:46:18 +08:00
|
|
|
if (ssk) {
|
|
|
|
/* make sure retransmit timer is
|
|
|
|
* running before we wait for memory.
|
|
|
|
*
|
|
|
|
* The retransmit timer might be needed
|
|
|
|
* to make the peer send an up-to-date
|
|
|
|
* MPTCP Ack.
|
|
|
|
*/
|
|
|
|
mptcp_set_timeout(sk, ssk);
|
|
|
|
if (!mptcp_timer_pending(sk))
|
|
|
|
mptcp_reset_timer(sk);
|
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:07 +08:00
|
|
|
mptcp_nospace(msk);
|
2020-03-28 05:48:39 +08:00
|
|
|
ret = sk_stream_wait_memory(sk, &timeo);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
mptcp_clean_una(sk);
|
|
|
|
|
2020-09-14 16:01:10 +08:00
|
|
|
ssk = mptcp_subflow_get_send(msk, &sndbuf);
|
2020-03-28 05:48:39 +08:00
|
|
|
if (list_empty(&msk->conn_list)) {
|
|
|
|
ret = -ENOTCONN;
|
|
|
|
goto out;
|
|
|
|
}
|
2020-01-22 08:56:18 +08:00
|
|
|
}
|
|
|
|
|
2020-09-14 16:01:10 +08:00
|
|
|
/* do auto tuning */
|
|
|
|
if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) &&
|
|
|
|
sndbuf > READ_ONCE(sk->sk_sndbuf))
|
|
|
|
WRITE_ONCE(sk->sk_sndbuf, sndbuf);
|
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
pr_debug("conn_list->subflow=%p", ssk);
|
2020-01-22 08:56:18 +08:00
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
lock_sock(ssk);
|
2020-05-16 16:46:19 +08:00
|
|
|
tx_ok = msg_data_left(msg);
|
|
|
|
while (tx_ok) {
|
2020-03-28 05:48:47 +08:00
|
|
|
ret = mptcp_sendmsg_frag(sk, ssk, msg, NULL, &timeo, &mss_now,
|
2020-01-22 08:56:27 +08:00
|
|
|
&size_goal);
|
2020-05-16 16:46:19 +08:00
|
|
|
if (ret < 0) {
|
|
|
|
if (ret == -EAGAIN && timeo > 0) {
|
|
|
|
mptcp_set_timeout(sk, ssk);
|
|
|
|
release_sock(ssk);
|
|
|
|
goto restart;
|
|
|
|
}
|
2020-01-22 08:56:23 +08:00
|
|
|
break;
|
2020-05-16 16:46:19 +08:00
|
|
|
}
|
2020-01-22 08:56:23 +08:00
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
/* burst can be negative, we will try move to the next subflow
|
|
|
|
* at selection time, if possible.
|
|
|
|
*/
|
|
|
|
msk->snd_burst -= ret;
|
2020-01-22 08:56:23 +08:00
|
|
|
copied += ret;
|
2020-05-16 16:46:18 +08:00
|
|
|
|
2020-05-16 16:46:19 +08:00
|
|
|
tx_ok = msg_data_left(msg);
|
|
|
|
if (!tx_ok)
|
|
|
|
break;
|
|
|
|
|
2020-05-16 16:46:20 +08:00
|
|
|
if (!sk_stream_memory_free(ssk) ||
|
2020-05-16 16:46:21 +08:00
|
|
|
!mptcp_page_frag_refill(ssk, pfrag) ||
|
2020-05-16 16:46:20 +08:00
|
|
|
!mptcp_ext_cache_refill(msk)) {
|
2020-05-16 16:46:19 +08:00
|
|
|
tcp_push(ssk, msg->msg_flags, mss_now,
|
|
|
|
tcp_sk(ssk)->nonagle, size_goal);
|
|
|
|
mptcp_set_timeout(sk, ssk);
|
|
|
|
release_sock(ssk);
|
|
|
|
goto restart;
|
|
|
|
}
|
|
|
|
|
2020-05-16 16:46:18 +08:00
|
|
|
/* memory is charged to mptcp level socket as well, i.e.
|
|
|
|
* if msg is very large, mptcp socket may run out of buffer
|
|
|
|
* space. mptcp_clean_una() will release data that has
|
|
|
|
* been acked at mptcp level in the mean time, so there is
|
|
|
|
* a good chance we can continue sending data right away.
|
2020-05-16 16:46:19 +08:00
|
|
|
*
|
|
|
|
* Normally, when the tcp subflow can accept more data, then
|
|
|
|
* so can the MPTCP socket. However, we need to cope with
|
|
|
|
* peers that might lag behind in their MPTCP-level
|
|
|
|
* acknowledgements, i.e. data might have been acked at
|
|
|
|
* tcp level only. So, we must also check the MPTCP socket
|
|
|
|
* limits before we send more data.
|
2020-05-16 16:46:18 +08:00
|
|
|
*/
|
|
|
|
if (unlikely(!sk_stream_memory_free(sk))) {
|
|
|
|
tcp_push(ssk, msg->msg_flags, mss_now,
|
|
|
|
tcp_sk(ssk)->nonagle, size_goal);
|
|
|
|
mptcp_clean_una(sk);
|
|
|
|
if (!sk_stream_memory_free(sk)) {
|
|
|
|
/* can't send more for now, need to wait for
|
|
|
|
* MPTCP-level ACKs from peer.
|
|
|
|
*
|
|
|
|
* Wakeup will happen via mptcp_clean_una().
|
|
|
|
*/
|
|
|
|
mptcp_set_timeout(sk, ssk);
|
|
|
|
release_sock(ssk);
|
2020-08-26 07:31:05 +08:00
|
|
|
goto restart;
|
2020-05-16 16:46:18 +08:00
|
|
|
}
|
|
|
|
}
|
2020-01-22 08:56:23 +08:00
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
mptcp_set_timeout(sk, ssk);
|
2020-01-22 08:56:27 +08:00
|
|
|
if (copied) {
|
|
|
|
tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle,
|
|
|
|
size_goal);
|
2020-03-28 05:48:44 +08:00
|
|
|
|
|
|
|
/* start the timer, if it's not pending */
|
|
|
|
if (!mptcp_timer_pending(sk))
|
|
|
|
mptcp_reset_timer(sk);
|
2020-01-22 08:56:27 +08:00
|
|
|
}
|
2020-01-22 08:56:23 +08:00
|
|
|
|
|
|
|
release_sock(ssk);
|
2020-02-29 07:47:39 +08:00
|
|
|
out:
|
2020-09-14 16:01:07 +08:00
|
|
|
ssk_check_wmem(msk);
|
2020-01-22 08:56:18 +08:00
|
|
|
release_sock(sk);
|
2020-08-04 00:40:39 +08:00
|
|
|
return copied ? : ret;
|
2020-01-22 08:56:15 +08:00
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:26 +08:00
|
|
|
static void mptcp_wait_data(struct sock *sk, long *timeo)
|
|
|
|
{
|
|
|
|
DEFINE_WAIT_FUNC(wait, woken_wake_function);
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
add_wait_queue(sk_sleep(sk), &wait);
|
|
|
|
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
|
|
|
|
|
|
|
|
sk_wait_event(sk, timeo,
|
|
|
|
test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait);
|
|
|
|
|
|
|
|
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
|
|
|
|
remove_wait_queue(sk_sleep(sk), &wait);
|
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
|
|
|
|
struct msghdr *msg,
|
|
|
|
size_t len)
|
|
|
|
{
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
int copied = 0;
|
|
|
|
|
|
|
|
while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
|
|
|
|
u32 offset = MPTCP_SKB_CB(skb)->offset;
|
|
|
|
u32 data_len = skb->len - offset;
|
|
|
|
u32 count = min_t(size_t, len - copied, data_len);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = skb_copy_datagram_msg(skb, offset, msg, count);
|
|
|
|
if (unlikely(err < 0)) {
|
|
|
|
if (!copied)
|
|
|
|
return err;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
copied += count;
|
|
|
|
|
|
|
|
if (count < data_len) {
|
|
|
|
MPTCP_SKB_CB(skb)->offset += count;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
__skb_unlink(skb, &sk->sk_receive_queue);
|
|
|
|
__kfree_skb(skb);
|
|
|
|
|
|
|
|
if (copied >= len)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
|
2020-07-01 03:24:45 +08:00
|
|
|
/* receive buffer autotuning. See tcp_rcv_space_adjust for more information.
|
|
|
|
*
|
|
|
|
* Only difference: Use highest rtt estimate of the subflows in use.
|
|
|
|
*/
|
|
|
|
static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
u32 time, advmss = 1;
|
|
|
|
u64 rtt_us, mstamp;
|
|
|
|
|
|
|
|
sock_owned_by_me(sk);
|
|
|
|
|
|
|
|
if (copied <= 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
msk->rcvq_space.copied += copied;
|
|
|
|
|
|
|
|
mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC);
|
|
|
|
time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time);
|
|
|
|
|
|
|
|
rtt_us = msk->rcvq_space.rtt_us;
|
|
|
|
if (rtt_us && time < (rtt_us >> 3))
|
|
|
|
return;
|
|
|
|
|
|
|
|
rtt_us = 0;
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
const struct tcp_sock *tp;
|
|
|
|
u64 sf_rtt_us;
|
|
|
|
u32 sf_advmss;
|
|
|
|
|
|
|
|
tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));
|
|
|
|
|
|
|
|
sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us);
|
|
|
|
sf_advmss = READ_ONCE(tp->advmss);
|
|
|
|
|
|
|
|
rtt_us = max(sf_rtt_us, rtt_us);
|
|
|
|
advmss = max(sf_advmss, advmss);
|
|
|
|
}
|
|
|
|
|
|
|
|
msk->rcvq_space.rtt_us = rtt_us;
|
|
|
|
if (time < (rtt_us >> 3) || rtt_us == 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (msk->rcvq_space.copied <= msk->rcvq_space.space)
|
|
|
|
goto new_measure;
|
|
|
|
|
|
|
|
if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf &&
|
|
|
|
!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
|
|
|
|
int rcvmem, rcvbuf;
|
|
|
|
u64 rcvwin, grow;
|
|
|
|
|
|
|
|
rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss;
|
|
|
|
|
|
|
|
grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space);
|
|
|
|
|
|
|
|
do_div(grow, msk->rcvq_space.space);
|
|
|
|
rcvwin += (grow << 1);
|
|
|
|
|
|
|
|
rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER);
|
|
|
|
while (tcp_win_from_space(sk, rcvmem) < advmss)
|
|
|
|
rcvmem += 128;
|
|
|
|
|
|
|
|
do_div(rcvwin, advmss);
|
|
|
|
rcvbuf = min_t(u64, rcvwin * rcvmem,
|
|
|
|
sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
|
|
|
|
|
|
|
|
if (rcvbuf > sk->sk_rcvbuf) {
|
|
|
|
u32 window_clamp;
|
|
|
|
|
|
|
|
window_clamp = tcp_win_from_space(sk, rcvbuf);
|
|
|
|
WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
|
|
|
|
|
|
|
|
/* Make subflows follow along. If we do not do this, we
|
|
|
|
* get drops at subflow level if skbs can't be moved to
|
|
|
|
* the mptcp rx queue fast enough (announced rcv_win can
|
|
|
|
* exceed ssk->sk_rcvbuf).
|
|
|
|
*/
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *ssk;
|
2020-09-14 16:01:18 +08:00
|
|
|
bool slow;
|
2020-07-01 03:24:45 +08:00
|
|
|
|
|
|
|
ssk = mptcp_subflow_tcp_sock(subflow);
|
2020-09-14 16:01:18 +08:00
|
|
|
slow = lock_sock_fast(ssk);
|
2020-07-01 03:24:45 +08:00
|
|
|
WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf);
|
|
|
|
tcp_sk(ssk)->window_clamp = window_clamp;
|
2020-09-14 16:01:18 +08:00
|
|
|
tcp_cleanup_rbuf(ssk, 1);
|
|
|
|
unlock_sock_fast(ssk, slow);
|
2020-07-01 03:24:45 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
msk->rcvq_space.space = msk->rcvq_space.copied;
|
|
|
|
new_measure:
|
|
|
|
msk->rcvq_space.copied = 0;
|
|
|
|
msk->rcvq_space.time = mstamp;
|
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
static bool __mptcp_move_skbs(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
unsigned int moved = 0;
|
|
|
|
bool done;
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
/* avoid looping forever below on racing close */
|
|
|
|
if (((struct sock *)msk)->sk_state == TCP_CLOSE)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
__mptcp_flush_join_list(msk);
|
2020-02-26 17:14:48 +08:00
|
|
|
do {
|
|
|
|
struct sock *ssk = mptcp_subflow_recv_lookup(msk);
|
|
|
|
|
|
|
|
if (!ssk)
|
|
|
|
break;
|
|
|
|
|
|
|
|
lock_sock(ssk);
|
|
|
|
done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
|
|
|
|
release_sock(ssk);
|
|
|
|
} while (!done);
|
|
|
|
|
2020-09-14 16:01:12 +08:00
|
|
|
if (mptcp_ofo_queue(msk) || moved > 0) {
|
|
|
|
mptcp_check_data_fin((struct sock *)msk);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
2020-02-26 17:14:48 +08:00
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
|
|
|
|
int nonblock, int flags, int *addr_len)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
2020-01-22 08:56:18 +08:00
|
|
|
int copied = 0;
|
2020-01-22 08:56:26 +08:00
|
|
|
int target;
|
|
|
|
long timeo;
|
2020-01-22 08:56:15 +08:00
|
|
|
|
|
|
|
if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT))
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
lock_sock(sk);
|
2020-01-22 08:56:26 +08:00
|
|
|
timeo = sock_rcvtimeo(sk, nonblock);
|
|
|
|
|
|
|
|
len = min_t(size_t, len, INT_MAX);
|
|
|
|
target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
|
2020-03-28 05:48:40 +08:00
|
|
|
__mptcp_flush_join_list(msk);
|
2020-01-22 08:56:26 +08:00
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
while (len > (size_t)copied) {
|
2020-01-22 08:56:26 +08:00
|
|
|
int bytes_read;
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied);
|
|
|
|
if (unlikely(bytes_read < 0)) {
|
|
|
|
if (!copied)
|
|
|
|
copied = bytes_read;
|
|
|
|
goto out_err;
|
|
|
|
}
|
2020-01-22 08:56:26 +08:00
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
copied += bytes_read;
|
2020-01-22 08:56:26 +08:00
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
if (skb_queue_empty(&sk->sk_receive_queue) &&
|
|
|
|
__mptcp_move_skbs(msk))
|
|
|
|
continue;
|
2020-01-22 08:56:26 +08:00
|
|
|
|
|
|
|
/* only the master socket status is relevant here. The exit
|
|
|
|
* conditions mirror closely tcp_recvmsg()
|
|
|
|
*/
|
|
|
|
if (copied >= target)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (copied) {
|
|
|
|
if (sk->sk_err ||
|
|
|
|
sk->sk_state == TCP_CLOSE ||
|
|
|
|
(sk->sk_shutdown & RCV_SHUTDOWN) ||
|
|
|
|
!timeo ||
|
|
|
|
signal_pending(current))
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
if (sk->sk_err) {
|
|
|
|
copied = sock_error(sk);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2020-06-10 16:47:41 +08:00
|
|
|
if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags))
|
|
|
|
mptcp_check_for_eof(msk);
|
|
|
|
|
2020-01-22 08:56:26 +08:00
|
|
|
if (sk->sk_shutdown & RCV_SHUTDOWN)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (sk->sk_state == TCP_CLOSE) {
|
|
|
|
copied = -ENOTCONN;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!timeo) {
|
|
|
|
copied = -EAGAIN;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (signal_pending(current)) {
|
|
|
|
copied = sock_intr_errno(timeo);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pr_debug("block timeout %ld", timeo);
|
|
|
|
mptcp_wait_data(sk, &timeo);
|
2020-01-22 08:56:18 +08:00
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
if (skb_queue_empty(&sk->sk_receive_queue)) {
|
|
|
|
/* entire backlog drained, clear DATA_READY. */
|
2020-01-22 08:56:26 +08:00
|
|
|
clear_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-01-22 08:56:18 +08:00
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
/* .. race-breaker: ssk might have gotten new data
|
|
|
|
* after last __mptcp_move_skbs() returned false.
|
2020-01-22 08:56:26 +08:00
|
|
|
*/
|
2020-02-26 17:14:48 +08:00
|
|
|
if (unlikely(__mptcp_move_skbs(msk)))
|
2020-01-22 08:56:26 +08:00
|
|
|
set_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-02-26 17:14:48 +08:00
|
|
|
} else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) {
|
|
|
|
/* data to read but mptcp_wait_data() cleared DATA_READY */
|
|
|
|
set_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-01-22 08:56:26 +08:00
|
|
|
}
|
2020-02-26 17:14:48 +08:00
|
|
|
out_err:
|
2020-09-14 16:01:09 +08:00
|
|
|
pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d",
|
|
|
|
msk, test_bit(MPTCP_DATA_READY, &msk->flags),
|
|
|
|
skb_queue_empty(&sk->sk_receive_queue), copied);
|
2020-07-01 03:24:45 +08:00
|
|
|
mptcp_rcv_space_adjust(msk, copied);
|
|
|
|
|
2020-01-22 08:56:26 +08:00
|
|
|
release_sock(sk);
|
2020-01-22 08:56:18 +08:00
|
|
|
return copied;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
static void mptcp_retransmit_handler(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
2020-07-29 06:12:09 +08:00
|
|
|
if (atomic64_read(&msk->snd_una) == READ_ONCE(msk->write_seq)) {
|
2020-03-28 05:48:44 +08:00
|
|
|
mptcp_stop_timer(sk);
|
2020-03-28 05:48:48 +08:00
|
|
|
} else {
|
|
|
|
set_bit(MPTCP_WORK_RTX, &msk->flags);
|
|
|
|
if (schedule_work(&msk->work))
|
|
|
|
sock_hold(sk);
|
|
|
|
}
|
2020-03-28 05:48:44 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void mptcp_retransmit_timer(struct timer_list *t)
|
|
|
|
{
|
|
|
|
struct inet_connection_sock *icsk = from_timer(icsk, t,
|
|
|
|
icsk_retransmit_timer);
|
|
|
|
struct sock *sk = &icsk->icsk_inet.sk;
|
|
|
|
|
|
|
|
bh_lock_sock(sk);
|
|
|
|
if (!sock_owned_by_user(sk)) {
|
|
|
|
mptcp_retransmit_handler(sk);
|
|
|
|
} else {
|
|
|
|
/* delegate our work to tcp_release_cb() */
|
|
|
|
if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED,
|
|
|
|
&sk->sk_tsq_flags))
|
|
|
|
sock_hold(sk);
|
|
|
|
}
|
|
|
|
bh_unlock_sock(sk);
|
|
|
|
sock_put(sk);
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:48 +08:00
|
|
|
/* Find an idle subflow. Return NULL if there is unacked data at tcp
|
|
|
|
* level.
|
|
|
|
*
|
|
|
|
* A backup subflow is returned only if that is the only kind available.
|
|
|
|
*/
|
|
|
|
static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct sock *backup = NULL;
|
|
|
|
|
|
|
|
sock_owned_by_me((const struct sock *)msk);
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
if (__mptcp_check_fallback(msk))
|
|
|
|
return msk->first;
|
|
|
|
|
2020-03-28 05:48:48 +08:00
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
2020-09-14 16:01:17 +08:00
|
|
|
if (!mptcp_subflow_active(subflow))
|
|
|
|
continue;
|
|
|
|
|
2020-03-28 05:48:48 +08:00
|
|
|
/* still data outstanding at TCP level? Don't retransmit. */
|
|
|
|
if (!tcp_write_queue_empty(ssk))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (subflow->backup) {
|
|
|
|
if (!backup)
|
|
|
|
backup = ssk;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
return ssk;
|
|
|
|
}
|
|
|
|
|
|
|
|
return backup;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
/* subflow sockets can be either outgoing (connect) or incoming
|
|
|
|
* (accept).
|
|
|
|
*
|
|
|
|
* Outgoing subflows use in-kernel sockets.
|
|
|
|
* Incoming subflows do not have their own 'struct socket' allocated,
|
|
|
|
* so we need to use tcp_close() after detaching them from the mptcp
|
|
|
|
* parent socket.
|
|
|
|
*/
|
2020-09-24 08:29:49 +08:00
|
|
|
void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
|
|
|
|
struct mptcp_subflow_context *subflow,
|
|
|
|
long timeout)
|
2020-01-22 08:56:18 +08:00
|
|
|
{
|
|
|
|
struct socket *sock = READ_ONCE(ssk->sk_socket);
|
|
|
|
|
|
|
|
list_del(&subflow->node);
|
|
|
|
|
|
|
|
if (sock && sock != sk->sk_socket) {
|
|
|
|
/* outgoing subflow */
|
|
|
|
sock_release(sock);
|
|
|
|
} else {
|
|
|
|
/* incoming subflow */
|
|
|
|
tcp_close(ssk, timeout);
|
|
|
|
}
|
2020-01-22 08:56:15 +08:00
|
|
|
}
|
|
|
|
|
2020-02-26 19:19:03 +08:00
|
|
|
static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-07-07 20:40:48 +08:00
|
|
|
static void pm_work(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct mptcp_pm_data *pm = &msk->pm;
|
|
|
|
|
|
|
|
spin_lock_bh(&msk->pm.lock);
|
|
|
|
|
|
|
|
pr_debug("msk=%p status=%x", msk, pm->status);
|
|
|
|
if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) {
|
|
|
|
pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED);
|
|
|
|
mptcp_pm_nl_add_addr_received(msk);
|
|
|
|
}
|
2020-09-24 08:29:49 +08:00
|
|
|
if (pm->status & BIT(MPTCP_PM_RM_ADDR_RECEIVED)) {
|
|
|
|
pm->status &= ~BIT(MPTCP_PM_RM_ADDR_RECEIVED);
|
|
|
|
mptcp_pm_nl_rm_addr_received(msk);
|
|
|
|
}
|
2020-07-07 20:40:48 +08:00
|
|
|
if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) {
|
|
|
|
pm->status &= ~BIT(MPTCP_PM_ESTABLISHED);
|
|
|
|
mptcp_pm_nl_fully_established(msk);
|
|
|
|
}
|
|
|
|
if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) {
|
|
|
|
pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED);
|
|
|
|
mptcp_pm_nl_subflow_established(msk);
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_bh(&msk->pm.lock);
|
|
|
|
}
|
|
|
|
|
2020-10-10 01:00:01 +08:00
|
|
|
static void __mptcp_close_subflow(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow, *tmp;
|
|
|
|
|
|
|
|
list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
|
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
|
|
|
if (inet_sk_state_load(ssk) != TCP_CLOSE)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
__mptcp_close_ssk((struct sock *)msk, ssk, subflow, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:47 +08:00
|
|
|
static void mptcp_worker(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
|
2020-03-28 05:48:48 +08:00
|
|
|
struct sock *ssk, *sk = &msk->sk.icsk_inet.sk;
|
2020-05-16 16:46:20 +08:00
|
|
|
int orig_len, orig_offset, mss_now = 0, size_goal = 0;
|
2020-03-28 05:48:48 +08:00
|
|
|
struct mptcp_data_frag *dfrag;
|
|
|
|
u64 orig_write_seq;
|
|
|
|
size_t copied = 0;
|
2020-08-17 05:14:20 +08:00
|
|
|
struct msghdr msg = {
|
|
|
|
.msg_flags = MSG_DONTWAIT,
|
|
|
|
};
|
2020-03-28 05:48:48 +08:00
|
|
|
long timeo = 0;
|
2020-02-26 17:14:47 +08:00
|
|
|
|
|
|
|
lock_sock(sk);
|
2020-03-28 05:48:48 +08:00
|
|
|
mptcp_clean_una(sk);
|
2020-07-29 06:12:06 +08:00
|
|
|
mptcp_check_data_fin_ack(sk);
|
2020-03-28 05:48:40 +08:00
|
|
|
__mptcp_flush_join_list(msk);
|
2020-10-10 01:00:01 +08:00
|
|
|
if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
|
|
|
|
__mptcp_close_subflow(msk);
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
__mptcp_move_skbs(msk);
|
2020-03-28 05:48:48 +08:00
|
|
|
|
2020-07-07 20:40:48 +08:00
|
|
|
if (msk->pm.status)
|
|
|
|
pm_work(msk);
|
|
|
|
|
2020-04-02 19:44:52 +08:00
|
|
|
if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags))
|
|
|
|
mptcp_check_for_eof(msk);
|
|
|
|
|
2020-07-29 06:12:06 +08:00
|
|
|
mptcp_check_data_fin(sk);
|
|
|
|
|
2020-03-28 05:48:48 +08:00
|
|
|
if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags))
|
|
|
|
goto unlock;
|
|
|
|
|
|
|
|
dfrag = mptcp_rtx_head(sk);
|
|
|
|
if (!dfrag)
|
|
|
|
goto unlock;
|
|
|
|
|
2020-05-16 16:46:20 +08:00
|
|
|
if (!mptcp_ext_cache_refill(msk))
|
|
|
|
goto reset_unlock;
|
|
|
|
|
2020-03-28 05:48:48 +08:00
|
|
|
ssk = mptcp_subflow_get_retrans(msk);
|
|
|
|
if (!ssk)
|
|
|
|
goto reset_unlock;
|
|
|
|
|
|
|
|
lock_sock(ssk);
|
|
|
|
|
|
|
|
orig_len = dfrag->data_len;
|
|
|
|
orig_offset = dfrag->offset;
|
|
|
|
orig_write_seq = dfrag->data_seq;
|
|
|
|
while (dfrag->data_len > 0) {
|
2020-05-16 16:46:20 +08:00
|
|
|
int ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo,
|
|
|
|
&mss_now, &size_goal);
|
2020-03-28 05:48:48 +08:00
|
|
|
if (ret < 0)
|
|
|
|
break;
|
|
|
|
|
2020-03-28 05:48:50 +08:00
|
|
|
MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS);
|
2020-03-28 05:48:48 +08:00
|
|
|
copied += ret;
|
|
|
|
dfrag->data_len -= ret;
|
|
|
|
dfrag->offset += ret;
|
2020-05-16 16:46:20 +08:00
|
|
|
|
|
|
|
if (!mptcp_ext_cache_refill(msk))
|
|
|
|
break;
|
2020-03-28 05:48:48 +08:00
|
|
|
}
|
|
|
|
if (copied)
|
|
|
|
tcp_push(ssk, msg.msg_flags, mss_now, tcp_sk(ssk)->nonagle,
|
|
|
|
size_goal);
|
|
|
|
|
|
|
|
dfrag->data_seq = orig_write_seq;
|
|
|
|
dfrag->offset = orig_offset;
|
|
|
|
dfrag->data_len = orig_len;
|
|
|
|
|
|
|
|
mptcp_set_timeout(sk, ssk);
|
|
|
|
release_sock(ssk);
|
|
|
|
|
|
|
|
reset_unlock:
|
|
|
|
if (!mptcp_timer_pending(sk))
|
|
|
|
mptcp_reset_timer(sk);
|
|
|
|
|
|
|
|
unlock:
|
2020-02-26 17:14:47 +08:00
|
|
|
release_sock(sk);
|
|
|
|
sock_put(sk);
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:28 +08:00
|
|
|
static int __mptcp_init_sock(struct sock *sk)
|
2020-01-22 08:56:15 +08:00
|
|
|
{
|
2020-01-22 08:56:18 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
2020-03-28 05:48:40 +08:00
|
|
|
spin_lock_init(&msk->join_list_lock);
|
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
INIT_LIST_HEAD(&msk->conn_list);
|
2020-03-28 05:48:40 +08:00
|
|
|
INIT_LIST_HEAD(&msk->join_list);
|
2020-03-28 05:48:43 +08:00
|
|
|
INIT_LIST_HEAD(&msk->rtx_queue);
|
2020-01-22 08:56:25 +08:00
|
|
|
__set_bit(MPTCP_SEND_SPACE, &msk->flags);
|
2020-02-26 17:14:47 +08:00
|
|
|
INIT_WORK(&msk->work, mptcp_worker);
|
2020-09-14 16:01:12 +08:00
|
|
|
msk->out_of_order_queue = RB_ROOT;
|
2020-01-22 08:56:18 +08:00
|
|
|
|
mptcp: cope with later TCP fallback
With MPTCP v1, passive connections can fallback to TCP after the
subflow becomes established:
syn + MP_CAPABLE ->
<- syn, ack + MP_CAPABLE
ack, seq = 3 ->
// OoO packet is accepted because in-sequence
// passive socket is created, is in ESTABLISHED
// status and tentatively as MP_CAPABLE
ack, seq = 2 ->
// no MP_CAPABLE opt, subflow should fallback to TCP
We can't use the 'subflow' socket fallback, as we don't have
it available for passive connection.
Instead, when the fallback is detected, replace the mptcp
socket with the underlying TCP subflow. Beyond covering
the above scenario, it makes a TCP fallback socket as efficient
as plain TCP ones.
Co-developed-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-22 08:56:33 +08:00
|
|
|
msk->first = NULL;
|
2020-02-26 19:19:03 +08:00
|
|
|
inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
|
mptcp: cope with later TCP fallback
With MPTCP v1, passive connections can fallback to TCP after the
subflow becomes established:
syn + MP_CAPABLE ->
<- syn, ack + MP_CAPABLE
ack, seq = 3 ->
// OoO packet is accepted because in-sequence
// passive socket is created, is in ESTABLISHED
// status and tentatively as MP_CAPABLE
ack, seq = 2 ->
// no MP_CAPABLE opt, subflow should fallback to TCP
We can't use the 'subflow' socket fallback, as we don't have
it available for passive connection.
Instead, when the fallback is detected, replace the mptcp
socket with the underlying TCP subflow. Beyond covering
the above scenario, it makes a TCP fallback socket as efficient
as plain TCP ones.
Co-developed-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-22 08:56:33 +08:00
|
|
|
|
2020-03-28 05:48:38 +08:00
|
|
|
mptcp_pm_data_init(msk);
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
/* re-use the csk retrans timer for MPTCP-level retrans */
|
|
|
|
timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);
|
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:28 +08:00
|
|
|
static int mptcp_init_sock(struct sock *sk)
|
|
|
|
{
|
2020-03-28 05:48:50 +08:00
|
|
|
struct net *net = sock_net(sk);
|
|
|
|
int ret;
|
2020-03-28 05:48:43 +08:00
|
|
|
|
2020-09-24 08:29:54 +08:00
|
|
|
ret = __mptcp_init_sock(sk);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2020-03-28 05:48:50 +08:00
|
|
|
if (!mptcp_is_enabled(net))
|
|
|
|
return -ENOPROTOOPT;
|
|
|
|
|
|
|
|
if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net))
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2020-06-30 04:26:23 +08:00
|
|
|
ret = __mptcp_socket_create(mptcp_sk(sk));
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2020-03-28 05:48:45 +08:00
|
|
|
sk_sockets_allocated_inc(sk);
|
2020-07-01 03:24:45 +08:00
|
|
|
sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1];
|
2020-09-14 16:01:10 +08:00
|
|
|
sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1];
|
2020-03-28 05:48:45 +08:00
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __mptcp_clear_xmit(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
struct mptcp_data_frag *dtmp, *dfrag;
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer);
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)
|
2020-03-28 05:48:45 +08:00
|
|
|
dfrag_clear(sk, dfrag);
|
2020-01-22 08:56:28 +08:00
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:47 +08:00
|
|
|
static void mptcp_cancel_work(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
if (cancel_work_sync(&msk->work))
|
|
|
|
sock_put(sk);
|
|
|
|
}
|
|
|
|
|
2020-09-24 08:29:49 +08:00
|
|
|
void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)
|
2020-01-22 08:56:21 +08:00
|
|
|
{
|
|
|
|
lock_sock(ssk);
|
|
|
|
|
|
|
|
switch (ssk->sk_state) {
|
|
|
|
case TCP_LISTEN:
|
|
|
|
if (!(how & RCV_SHUTDOWN))
|
|
|
|
break;
|
2020-08-24 06:36:59 +08:00
|
|
|
fallthrough;
|
2020-01-22 08:56:21 +08:00
|
|
|
case TCP_SYN_SENT:
|
|
|
|
tcp_disconnect(ssk, O_NONBLOCK);
|
|
|
|
break;
|
|
|
|
default:
|
2020-07-29 06:12:06 +08:00
|
|
|
if (__mptcp_check_fallback(mptcp_sk(sk))) {
|
|
|
|
pr_debug("Fallback");
|
|
|
|
ssk->sk_shutdown |= how;
|
|
|
|
tcp_shutdown(ssk, how);
|
|
|
|
} else {
|
|
|
|
pr_debug("Sending DATA_FIN on subflow %p", ssk);
|
|
|
|
mptcp_set_timeout(sk, ssk);
|
|
|
|
tcp_send_ack(ssk);
|
|
|
|
}
|
2020-01-22 08:56:21 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
release_sock(ssk);
|
|
|
|
}
|
|
|
|
|
2020-07-29 06:12:04 +08:00
|
|
|
static const unsigned char new_state[16] = {
|
|
|
|
/* current state: new state: action: */
|
|
|
|
[0 /* (Invalid) */] = TCP_CLOSE,
|
|
|
|
[TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
|
|
|
|
[TCP_SYN_SENT] = TCP_CLOSE,
|
|
|
|
[TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
|
|
|
|
[TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
|
|
|
|
[TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
|
|
|
|
[TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */
|
|
|
|
[TCP_CLOSE] = TCP_CLOSE,
|
|
|
|
[TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
|
|
|
|
[TCP_LAST_ACK] = TCP_LAST_ACK,
|
|
|
|
[TCP_LISTEN] = TCP_CLOSE,
|
|
|
|
[TCP_CLOSING] = TCP_CLOSING,
|
|
|
|
[TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */
|
|
|
|
};
|
|
|
|
|
|
|
|
static int mptcp_close_state(struct sock *sk)
|
|
|
|
{
|
|
|
|
int next = (int)new_state[sk->sk_state];
|
|
|
|
int ns = next & TCP_STATE_MASK;
|
|
|
|
|
|
|
|
inet_sk_state_store(sk, ns);
|
|
|
|
|
|
|
|
return next & TCP_ACTION_FIN;
|
|
|
|
}
|
|
|
|
|
2020-02-05 01:12:30 +08:00
|
|
|
static void mptcp_close(struct sock *sk, long timeout)
|
2020-01-22 08:56:15 +08:00
|
|
|
{
|
2020-01-22 08:56:18 +08:00
|
|
|
struct mptcp_subflow_context *subflow, *tmp;
|
2020-01-22 08:56:15 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
mptcp: avoid a lockdep splat when mcast group was joined
syzbot triggered following lockdep splat:
ffffffff82d2cd40 (rtnl_mutex){+.+.}, at: ip_mc_drop_socket+0x52/0x180
but task is already holding lock:
ffff8881187a2310 (sk_lock-AF_INET){+.+.}, at: mptcp_close+0x18/0x30
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (sk_lock-AF_INET){+.+.}:
lock_acquire+0xee/0x230
lock_sock_nested+0x89/0xc0
do_ip_setsockopt.isra.0+0x335/0x22f0
ip_setsockopt+0x35/0x60
tcp_setsockopt+0x5d/0x90
__sys_setsockopt+0xf3/0x190
__x64_sys_setsockopt+0x61/0x70
do_syscall_64+0x72/0x300
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #0 (rtnl_mutex){+.+.}:
check_prevs_add+0x2b7/0x1210
__lock_acquire+0x10b6/0x1400
lock_acquire+0xee/0x230
__mutex_lock+0x120/0xc70
ip_mc_drop_socket+0x52/0x180
inet_release+0x36/0xe0
__sock_release+0xfd/0x130
__mptcp_close+0xa8/0x1f0
inet_release+0x7f/0xe0
__sock_release+0x69/0x130
sock_close+0x18/0x20
__fput+0x179/0x400
task_work_run+0xd5/0x110
do_exit+0x685/0x1510
do_group_exit+0x7e/0x170
__x64_sys_exit_group+0x28/0x30
do_syscall_64+0x72/0x300
entry_SYSCALL_64_after_hwframe+0x49/0xbe
The trigger is:
socket(AF_INET, SOCK_STREAM, 0x106 /* IPPROTO_MPTCP */) = 4
setsockopt(4, SOL_IP, MCAST_JOIN_GROUP, {gr_interface=7, gr_group={sa_family=AF_INET, sin_port=htons(20003), sin_addr=inet_addr("224.0.0.2")}}, 136) = 0
exit(0)
Which results in a call to rtnl_lock while we are holding
the parent mptcp socket lock via
mptcp_close -> lock_sock(msk) -> inet_release -> ip_mc_drop_socket -> rtnl_lock().
>From lockdep point of view we thus have both
'rtnl_lock; lock_sock' and 'lock_sock; rtnl_lock'.
Fix this by stealing the msk conn_list and doing the subflow close
without holding the msk lock.
Fixes: cec37a6e41aae7bf ("mptcp: Handle MP_CAPABLE options for outgoing connections")
Reported-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-29 22:54:45 +08:00
|
|
|
LIST_HEAD(conn_list);
|
2020-01-22 08:56:15 +08:00
|
|
|
|
2020-02-05 01:12:30 +08:00
|
|
|
lock_sock(sk);
|
2020-07-29 06:12:06 +08:00
|
|
|
sk->sk_shutdown = SHUTDOWN_MASK;
|
|
|
|
|
|
|
|
if (sk->sk_state == TCP_LISTEN) {
|
|
|
|
inet_sk_state_store(sk, TCP_CLOSE);
|
|
|
|
goto cleanup;
|
|
|
|
} else if (sk->sk_state == TCP_CLOSE) {
|
|
|
|
goto cleanup;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (__mptcp_check_fallback(msk)) {
|
|
|
|
goto update_state;
|
|
|
|
} else if (mptcp_close_state(sk)) {
|
|
|
|
pr_debug("Sending DATA_FIN sk=%p", sk);
|
|
|
|
WRITE_ONCE(msk->write_seq, msk->write_seq + 1);
|
|
|
|
WRITE_ONCE(msk->snd_data_fin_enable, 1);
|
|
|
|
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
|
|
|
mptcp_subflow_shutdown(sk, tcp_sk, SHUTDOWN_MASK);
|
|
|
|
}
|
|
|
|
}
|
2020-02-05 01:12:30 +08:00
|
|
|
|
2020-07-29 06:12:06 +08:00
|
|
|
sk_stream_wait_close(sk, timeout);
|
|
|
|
|
|
|
|
update_state:
|
2020-01-22 08:56:15 +08:00
|
|
|
inet_sk_state_store(sk, TCP_CLOSE);
|
|
|
|
|
2020-07-29 06:12:06 +08:00
|
|
|
cleanup:
|
2020-05-29 23:43:30 +08:00
|
|
|
/* be sure to always acquire the join list lock, to sync vs
|
|
|
|
* mptcp_finish_join().
|
|
|
|
*/
|
|
|
|
spin_lock_bh(&msk->join_list_lock);
|
|
|
|
list_splice_tail_init(&msk->join_list, &msk->conn_list);
|
|
|
|
spin_unlock_bh(&msk->join_list_lock);
|
mptcp: avoid a lockdep splat when mcast group was joined
syzbot triggered following lockdep splat:
ffffffff82d2cd40 (rtnl_mutex){+.+.}, at: ip_mc_drop_socket+0x52/0x180
but task is already holding lock:
ffff8881187a2310 (sk_lock-AF_INET){+.+.}, at: mptcp_close+0x18/0x30
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (sk_lock-AF_INET){+.+.}:
lock_acquire+0xee/0x230
lock_sock_nested+0x89/0xc0
do_ip_setsockopt.isra.0+0x335/0x22f0
ip_setsockopt+0x35/0x60
tcp_setsockopt+0x5d/0x90
__sys_setsockopt+0xf3/0x190
__x64_sys_setsockopt+0x61/0x70
do_syscall_64+0x72/0x300
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #0 (rtnl_mutex){+.+.}:
check_prevs_add+0x2b7/0x1210
__lock_acquire+0x10b6/0x1400
lock_acquire+0xee/0x230
__mutex_lock+0x120/0xc70
ip_mc_drop_socket+0x52/0x180
inet_release+0x36/0xe0
__sock_release+0xfd/0x130
__mptcp_close+0xa8/0x1f0
inet_release+0x7f/0xe0
__sock_release+0x69/0x130
sock_close+0x18/0x20
__fput+0x179/0x400
task_work_run+0xd5/0x110
do_exit+0x685/0x1510
do_group_exit+0x7e/0x170
__x64_sys_exit_group+0x28/0x30
do_syscall_64+0x72/0x300
entry_SYSCALL_64_after_hwframe+0x49/0xbe
The trigger is:
socket(AF_INET, SOCK_STREAM, 0x106 /* IPPROTO_MPTCP */) = 4
setsockopt(4, SOL_IP, MCAST_JOIN_GROUP, {gr_interface=7, gr_group={sa_family=AF_INET, sin_port=htons(20003), sin_addr=inet_addr("224.0.0.2")}}, 136) = 0
exit(0)
Which results in a call to rtnl_lock while we are holding
the parent mptcp socket lock via
mptcp_close -> lock_sock(msk) -> inet_release -> ip_mc_drop_socket -> rtnl_lock().
>From lockdep point of view we thus have both
'rtnl_lock; lock_sock' and 'lock_sock; rtnl_lock'.
Fix this by stealing the msk conn_list and doing the subflow close
without holding the msk lock.
Fixes: cec37a6e41aae7bf ("mptcp: Handle MP_CAPABLE options for outgoing connections")
Reported-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-29 22:54:45 +08:00
|
|
|
list_splice_init(&msk->conn_list, &conn_list);
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
__mptcp_clear_xmit(sk);
|
|
|
|
|
mptcp: avoid a lockdep splat when mcast group was joined
syzbot triggered following lockdep splat:
ffffffff82d2cd40 (rtnl_mutex){+.+.}, at: ip_mc_drop_socket+0x52/0x180
but task is already holding lock:
ffff8881187a2310 (sk_lock-AF_INET){+.+.}, at: mptcp_close+0x18/0x30
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (sk_lock-AF_INET){+.+.}:
lock_acquire+0xee/0x230
lock_sock_nested+0x89/0xc0
do_ip_setsockopt.isra.0+0x335/0x22f0
ip_setsockopt+0x35/0x60
tcp_setsockopt+0x5d/0x90
__sys_setsockopt+0xf3/0x190
__x64_sys_setsockopt+0x61/0x70
do_syscall_64+0x72/0x300
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #0 (rtnl_mutex){+.+.}:
check_prevs_add+0x2b7/0x1210
__lock_acquire+0x10b6/0x1400
lock_acquire+0xee/0x230
__mutex_lock+0x120/0xc70
ip_mc_drop_socket+0x52/0x180
inet_release+0x36/0xe0
__sock_release+0xfd/0x130
__mptcp_close+0xa8/0x1f0
inet_release+0x7f/0xe0
__sock_release+0x69/0x130
sock_close+0x18/0x20
__fput+0x179/0x400
task_work_run+0xd5/0x110
do_exit+0x685/0x1510
do_group_exit+0x7e/0x170
__x64_sys_exit_group+0x28/0x30
do_syscall_64+0x72/0x300
entry_SYSCALL_64_after_hwframe+0x49/0xbe
The trigger is:
socket(AF_INET, SOCK_STREAM, 0x106 /* IPPROTO_MPTCP */) = 4
setsockopt(4, SOL_IP, MCAST_JOIN_GROUP, {gr_interface=7, gr_group={sa_family=AF_INET, sin_port=htons(20003), sin_addr=inet_addr("224.0.0.2")}}, 136) = 0
exit(0)
Which results in a call to rtnl_lock while we are holding
the parent mptcp socket lock via
mptcp_close -> lock_sock(msk) -> inet_release -> ip_mc_drop_socket -> rtnl_lock().
>From lockdep point of view we thus have both
'rtnl_lock; lock_sock' and 'lock_sock; rtnl_lock'.
Fix this by stealing the msk conn_list and doing the subflow close
without holding the msk lock.
Fixes: cec37a6e41aae7bf ("mptcp: Handle MP_CAPABLE options for outgoing connections")
Reported-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-29 22:54:45 +08:00
|
|
|
release_sock(sk);
|
|
|
|
|
|
|
|
list_for_each_entry_safe(subflow, tmp, &conn_list, node) {
|
2020-01-22 08:56:18 +08:00
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
__mptcp_close_ssk(sk, ssk, subflow, timeout);
|
2020-01-22 08:56:15 +08:00
|
|
|
}
|
|
|
|
|
2020-02-26 17:14:47 +08:00
|
|
|
mptcp_cancel_work(sk);
|
|
|
|
|
2020-02-26 17:14:48 +08:00
|
|
|
__skb_queue_purge(&sk->sk_receive_queue);
|
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
sk_common_release(sk);
|
2020-01-22 08:56:15 +08:00
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:19 +08:00
|
|
|
static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
|
|
|
|
{
|
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
|
|
|
const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
|
|
|
|
struct ipv6_pinfo *msk6 = inet6_sk(msk);
|
|
|
|
|
|
|
|
msk->sk_v6_daddr = ssk->sk_v6_daddr;
|
|
|
|
msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;
|
|
|
|
|
|
|
|
if (msk6 && ssk6) {
|
|
|
|
msk6->saddr = ssk6->saddr;
|
|
|
|
msk6->flow_label = ssk6->flow_label;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
|
|
|
|
inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
|
|
|
|
inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
|
|
|
|
inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
|
|
|
|
inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
|
|
|
|
inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:43 +08:00
|
|
|
static int mptcp_disconnect(struct sock *sk, int flags)
|
|
|
|
{
|
2020-04-30 02:43:20 +08:00
|
|
|
/* Should never be called.
|
|
|
|
* inet_stream_connect() calls ->disconnect, but that
|
|
|
|
* refers to the subflow socket, not the mptcp one.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return 0;
|
2020-03-28 05:48:43 +08:00
|
|
|
}
|
|
|
|
|
2020-02-06 07:39:37 +08:00
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
|
|
|
static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
|
|
|
|
{
|
|
|
|
unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo);
|
|
|
|
|
|
|
|
return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2020-04-20 22:25:06 +08:00
|
|
|
struct sock *mptcp_sk_clone(const struct sock *sk,
|
mptcp: move option parsing into mptcp_incoming_options()
The mptcp_options_received structure carries several per
packet flags (mp_capable, mp_join, etc.). Such fields must
be cleared on each packet, even on dropped ones or packet
not carrying any MPTCP options, but the current mptcp
code clears them only on TCP option reset.
On several races/corner cases we end-up with stray bits in
incoming options, leading to WARN_ON splats. e.g.:
[ 171.164906] Bad mapping: ssn=32714 map_seq=1 map_data_len=32713
[ 171.165006] WARNING: CPU: 1 PID: 5026 at net/mptcp/subflow.c:533 warn_bad_map (linux-mptcp/net/mptcp/subflow.c:533 linux-mptcp/net/mptcp/subflow.c:531)
[ 171.167632] Modules linked in: ip6_vti ip_vti ip_gre ipip sit tunnel4 ip_tunnel geneve ip6_udp_tunnel udp_tunnel macsec macvtap tap ipvlan macvlan 8021q garp mrp xfrm_interface veth netdevsim nlmon dummy team bonding vcan bridge stp llc ip6_gre gre ip6_tunnel tunnel6 tun binfmt_misc intel_rapl_msr intel_rapl_common rfkill kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel joydev virtio_balloon pcspkr i2c_piix4 sunrpc ip_tables xfs libcrc32c crc32c_intel serio_raw virtio_console ata_generic virtio_blk virtio_net net_failover failover ata_piix libata
[ 171.199464] CPU: 1 PID: 5026 Comm: repro Not tainted 5.7.0-rc1.mptcp_f227fdf5d388+ #95
[ 171.200886] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-2.fc30 04/01/2014
[ 171.202546] RIP: 0010:warn_bad_map (linux-mptcp/net/mptcp/subflow.c:533 linux-mptcp/net/mptcp/subflow.c:531)
[ 171.206537] Code: c1 ea 03 0f b6 14 02 48 89 f8 83 e0 07 83 c0 03 38 d0 7c 04 84 d2 75 1d 8b 55 3c 44 89 e6 48 c7 c7 20 51 13 95 e8 37 8b 22 fe <0f> 0b 48 83 c4 08 5b 5d 41 5c c3 89 4c 24 04 e8 db d6 94 fe 8b 4c
[ 171.220473] RSP: 0018:ffffc90000150560 EFLAGS: 00010282
[ 171.221639] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[ 171.223108] RDX: 0000000000000000 RSI: 0000000000000008 RDI: fffff5200002a09e
[ 171.224388] RBP: ffff8880aa6e3c00 R08: 0000000000000001 R09: fffffbfff2ec9955
[ 171.225706] R10: ffffffff9764caa7 R11: fffffbfff2ec9954 R12: 0000000000007fca
[ 171.227211] R13: ffff8881066f4a7f R14: ffff8880aa6e3c00 R15: 0000000000000020
[ 171.228460] FS: 00007f8623719740(0000) GS:ffff88810be00000(0000) knlGS:0000000000000000
[ 171.230065] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 171.231303] CR2: 00007ffdab190a50 CR3: 00000001038ea006 CR4: 0000000000160ee0
[ 171.232586] Call Trace:
[ 171.233109] <IRQ>
[ 171.233531] get_mapping_status (linux-mptcp/net/mptcp/subflow.c:691)
[ 171.234371] mptcp_subflow_data_available (linux-mptcp/net/mptcp/subflow.c:736 linux-mptcp/net/mptcp/subflow.c:832)
[ 171.238181] subflow_state_change (linux-mptcp/net/mptcp/subflow.c:1085 (discriminator 1))
[ 171.239066] tcp_fin (linux-mptcp/net/ipv4/tcp_input.c:4217)
[ 171.240123] tcp_data_queue (linux-mptcp/./include/linux/compiler.h:199 linux-mptcp/net/ipv4/tcp_input.c:4822)
[ 171.245083] tcp_rcv_established (linux-mptcp/./include/linux/skbuff.h:1785 linux-mptcp/./include/net/tcp.h:1774 linux-mptcp/./include/net/tcp.h:1847 linux-mptcp/net/ipv4/tcp_input.c:5238 linux-mptcp/net/ipv4/tcp_input.c:5730)
[ 171.254089] tcp_v4_rcv (linux-mptcp/./include/linux/spinlock.h:393 linux-mptcp/net/ipv4/tcp_ipv4.c:2009)
[ 171.258969] ip_protocol_deliver_rcu (linux-mptcp/net/ipv4/ip_input.c:204 (discriminator 1))
[ 171.260214] ip_local_deliver_finish (linux-mptcp/./include/linux/rcupdate.h:651 linux-mptcp/net/ipv4/ip_input.c:232)
[ 171.261389] ip_local_deliver (linux-mptcp/./include/linux/netfilter.h:307 linux-mptcp/./include/linux/netfilter.h:301 linux-mptcp/net/ipv4/ip_input.c:252)
[ 171.265884] ip_rcv (linux-mptcp/./include/linux/netfilter.h:307 linux-mptcp/./include/linux/netfilter.h:301 linux-mptcp/net/ipv4/ip_input.c:539)
[ 171.273666] process_backlog (linux-mptcp/./include/linux/rcupdate.h:651 linux-mptcp/net/core/dev.c:6135)
[ 171.275328] net_rx_action (linux-mptcp/net/core/dev.c:6572 linux-mptcp/net/core/dev.c:6640)
[ 171.280472] __do_softirq (linux-mptcp/./arch/x86/include/asm/jump_label.h:25 linux-mptcp/./include/linux/jump_label.h:200 linux-mptcp/./include/trace/events/irq.h:142 linux-mptcp/kernel/softirq.c:293)
[ 171.281379] do_softirq_own_stack (linux-mptcp/arch/x86/entry/entry_64.S:1083)
[ 171.282358] </IRQ>
We could address the issue clearing explicitly the relevant fields
in several places - tcp_parse_option, tcp_fast_parse_options,
possibly others.
Instead we move the MPTCP option parsing into the already existing
mptcp ingress hook, so that we need to clear the fields in a single
place.
This allows us dropping an MPTCP hook from the TCP code and
removing the quite large mptcp_options_received from the tcp_sock
struct. On the flip side, the MPTCP sockets will traverse the
option space twice (in tcp_parse_option() and in
mptcp_incoming_options(). That looks acceptable: we already
do that for syn and 3rd ack packets, plain TCP socket will
benefit from it, and even MPTCP sockets will experience better
code locality, reducing the jumps between TCP and MPTCP code.
v1 -> v2:
- rebased on current '-net' tree
Fixes: 648ef4b88673 ("mptcp: Implement MPTCP receive path")
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-04-30 21:01:52 +08:00
|
|
|
const struct mptcp_options_received *mp_opt,
|
2020-04-20 22:25:06 +08:00
|
|
|
struct request_sock *req)
|
2020-02-06 07:39:37 +08:00
|
|
|
{
|
2020-03-13 23:52:41 +08:00
|
|
|
struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
|
2020-02-06 07:39:37 +08:00
|
|
|
struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
|
2020-03-13 23:52:41 +08:00
|
|
|
struct mptcp_sock *msk;
|
|
|
|
u64 ack_seq;
|
2020-02-06 07:39:37 +08:00
|
|
|
|
|
|
|
if (!nsk)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
|
|
|
if (nsk->sk_family == AF_INET6)
|
|
|
|
inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk);
|
|
|
|
#endif
|
|
|
|
|
2020-03-13 23:52:41 +08:00
|
|
|
__mptcp_init_sock(nsk);
|
|
|
|
|
|
|
|
msk = mptcp_sk(nsk);
|
|
|
|
msk->local_key = subflow_req->local_key;
|
|
|
|
msk->token = subflow_req->token;
|
|
|
|
msk->subflow = NULL;
|
2020-07-23 19:02:32 +08:00
|
|
|
WRITE_ONCE(msk->fully_established, false);
|
2020-03-13 23:52:41 +08:00
|
|
|
|
|
|
|
msk->write_seq = subflow_req->idsn + 1;
|
2020-03-28 05:48:42 +08:00
|
|
|
atomic64_set(&msk->snd_una, msk->write_seq);
|
mptcp: move option parsing into mptcp_incoming_options()
The mptcp_options_received structure carries several per
packet flags (mp_capable, mp_join, etc.). Such fields must
be cleared on each packet, even on dropped ones or packet
not carrying any MPTCP options, but the current mptcp
code clears them only on TCP option reset.
On several races/corner cases we end-up with stray bits in
incoming options, leading to WARN_ON splats. e.g.:
[ 171.164906] Bad mapping: ssn=32714 map_seq=1 map_data_len=32713
[ 171.165006] WARNING: CPU: 1 PID: 5026 at net/mptcp/subflow.c:533 warn_bad_map (linux-mptcp/net/mptcp/subflow.c:533 linux-mptcp/net/mptcp/subflow.c:531)
[ 171.167632] Modules linked in: ip6_vti ip_vti ip_gre ipip sit tunnel4 ip_tunnel geneve ip6_udp_tunnel udp_tunnel macsec macvtap tap ipvlan macvlan 8021q garp mrp xfrm_interface veth netdevsim nlmon dummy team bonding vcan bridge stp llc ip6_gre gre ip6_tunnel tunnel6 tun binfmt_misc intel_rapl_msr intel_rapl_common rfkill kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel joydev virtio_balloon pcspkr i2c_piix4 sunrpc ip_tables xfs libcrc32c crc32c_intel serio_raw virtio_console ata_generic virtio_blk virtio_net net_failover failover ata_piix libata
[ 171.199464] CPU: 1 PID: 5026 Comm: repro Not tainted 5.7.0-rc1.mptcp_f227fdf5d388+ #95
[ 171.200886] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-2.fc30 04/01/2014
[ 171.202546] RIP: 0010:warn_bad_map (linux-mptcp/net/mptcp/subflow.c:533 linux-mptcp/net/mptcp/subflow.c:531)
[ 171.206537] Code: c1 ea 03 0f b6 14 02 48 89 f8 83 e0 07 83 c0 03 38 d0 7c 04 84 d2 75 1d 8b 55 3c 44 89 e6 48 c7 c7 20 51 13 95 e8 37 8b 22 fe <0f> 0b 48 83 c4 08 5b 5d 41 5c c3 89 4c 24 04 e8 db d6 94 fe 8b 4c
[ 171.220473] RSP: 0018:ffffc90000150560 EFLAGS: 00010282
[ 171.221639] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[ 171.223108] RDX: 0000000000000000 RSI: 0000000000000008 RDI: fffff5200002a09e
[ 171.224388] RBP: ffff8880aa6e3c00 R08: 0000000000000001 R09: fffffbfff2ec9955
[ 171.225706] R10: ffffffff9764caa7 R11: fffffbfff2ec9954 R12: 0000000000007fca
[ 171.227211] R13: ffff8881066f4a7f R14: ffff8880aa6e3c00 R15: 0000000000000020
[ 171.228460] FS: 00007f8623719740(0000) GS:ffff88810be00000(0000) knlGS:0000000000000000
[ 171.230065] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 171.231303] CR2: 00007ffdab190a50 CR3: 00000001038ea006 CR4: 0000000000160ee0
[ 171.232586] Call Trace:
[ 171.233109] <IRQ>
[ 171.233531] get_mapping_status (linux-mptcp/net/mptcp/subflow.c:691)
[ 171.234371] mptcp_subflow_data_available (linux-mptcp/net/mptcp/subflow.c:736 linux-mptcp/net/mptcp/subflow.c:832)
[ 171.238181] subflow_state_change (linux-mptcp/net/mptcp/subflow.c:1085 (discriminator 1))
[ 171.239066] tcp_fin (linux-mptcp/net/ipv4/tcp_input.c:4217)
[ 171.240123] tcp_data_queue (linux-mptcp/./include/linux/compiler.h:199 linux-mptcp/net/ipv4/tcp_input.c:4822)
[ 171.245083] tcp_rcv_established (linux-mptcp/./include/linux/skbuff.h:1785 linux-mptcp/./include/net/tcp.h:1774 linux-mptcp/./include/net/tcp.h:1847 linux-mptcp/net/ipv4/tcp_input.c:5238 linux-mptcp/net/ipv4/tcp_input.c:5730)
[ 171.254089] tcp_v4_rcv (linux-mptcp/./include/linux/spinlock.h:393 linux-mptcp/net/ipv4/tcp_ipv4.c:2009)
[ 171.258969] ip_protocol_deliver_rcu (linux-mptcp/net/ipv4/ip_input.c:204 (discriminator 1))
[ 171.260214] ip_local_deliver_finish (linux-mptcp/./include/linux/rcupdate.h:651 linux-mptcp/net/ipv4/ip_input.c:232)
[ 171.261389] ip_local_deliver (linux-mptcp/./include/linux/netfilter.h:307 linux-mptcp/./include/linux/netfilter.h:301 linux-mptcp/net/ipv4/ip_input.c:252)
[ 171.265884] ip_rcv (linux-mptcp/./include/linux/netfilter.h:307 linux-mptcp/./include/linux/netfilter.h:301 linux-mptcp/net/ipv4/ip_input.c:539)
[ 171.273666] process_backlog (linux-mptcp/./include/linux/rcupdate.h:651 linux-mptcp/net/core/dev.c:6135)
[ 171.275328] net_rx_action (linux-mptcp/net/core/dev.c:6572 linux-mptcp/net/core/dev.c:6640)
[ 171.280472] __do_softirq (linux-mptcp/./arch/x86/include/asm/jump_label.h:25 linux-mptcp/./include/linux/jump_label.h:200 linux-mptcp/./include/trace/events/irq.h:142 linux-mptcp/kernel/softirq.c:293)
[ 171.281379] do_softirq_own_stack (linux-mptcp/arch/x86/entry/entry_64.S:1083)
[ 171.282358] </IRQ>
We could address the issue clearing explicitly the relevant fields
in several places - tcp_parse_option, tcp_fast_parse_options,
possibly others.
Instead we move the MPTCP option parsing into the already existing
mptcp ingress hook, so that we need to clear the fields in a single
place.
This allows us dropping an MPTCP hook from the TCP code and
removing the quite large mptcp_options_received from the tcp_sock
struct. On the flip side, the MPTCP sockets will traverse the
option space twice (in tcp_parse_option() and in
mptcp_incoming_options(). That looks acceptable: we already
do that for syn and 3rd ack packets, plain TCP socket will
benefit from it, and even MPTCP sockets will experience better
code locality, reducing the jumps between TCP and MPTCP code.
v1 -> v2:
- rebased on current '-net' tree
Fixes: 648ef4b88673 ("mptcp: Implement MPTCP receive path")
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-04-30 21:01:52 +08:00
|
|
|
if (mp_opt->mp_capable) {
|
2020-03-13 23:52:41 +08:00
|
|
|
msk->can_ack = true;
|
mptcp: move option parsing into mptcp_incoming_options()
The mptcp_options_received structure carries several per
packet flags (mp_capable, mp_join, etc.). Such fields must
be cleared on each packet, even on dropped ones or packet
not carrying any MPTCP options, but the current mptcp
code clears them only on TCP option reset.
On several races/corner cases we end-up with stray bits in
incoming options, leading to WARN_ON splats. e.g.:
[ 171.164906] Bad mapping: ssn=32714 map_seq=1 map_data_len=32713
[ 171.165006] WARNING: CPU: 1 PID: 5026 at net/mptcp/subflow.c:533 warn_bad_map (linux-mptcp/net/mptcp/subflow.c:533 linux-mptcp/net/mptcp/subflow.c:531)
[ 171.167632] Modules linked in: ip6_vti ip_vti ip_gre ipip sit tunnel4 ip_tunnel geneve ip6_udp_tunnel udp_tunnel macsec macvtap tap ipvlan macvlan 8021q garp mrp xfrm_interface veth netdevsim nlmon dummy team bonding vcan bridge stp llc ip6_gre gre ip6_tunnel tunnel6 tun binfmt_misc intel_rapl_msr intel_rapl_common rfkill kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul ghash_clmulni_intel joydev virtio_balloon pcspkr i2c_piix4 sunrpc ip_tables xfs libcrc32c crc32c_intel serio_raw virtio_console ata_generic virtio_blk virtio_net net_failover failover ata_piix libata
[ 171.199464] CPU: 1 PID: 5026 Comm: repro Not tainted 5.7.0-rc1.mptcp_f227fdf5d388+ #95
[ 171.200886] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-2.fc30 04/01/2014
[ 171.202546] RIP: 0010:warn_bad_map (linux-mptcp/net/mptcp/subflow.c:533 linux-mptcp/net/mptcp/subflow.c:531)
[ 171.206537] Code: c1 ea 03 0f b6 14 02 48 89 f8 83 e0 07 83 c0 03 38 d0 7c 04 84 d2 75 1d 8b 55 3c 44 89 e6 48 c7 c7 20 51 13 95 e8 37 8b 22 fe <0f> 0b 48 83 c4 08 5b 5d 41 5c c3 89 4c 24 04 e8 db d6 94 fe 8b 4c
[ 171.220473] RSP: 0018:ffffc90000150560 EFLAGS: 00010282
[ 171.221639] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[ 171.223108] RDX: 0000000000000000 RSI: 0000000000000008 RDI: fffff5200002a09e
[ 171.224388] RBP: ffff8880aa6e3c00 R08: 0000000000000001 R09: fffffbfff2ec9955
[ 171.225706] R10: ffffffff9764caa7 R11: fffffbfff2ec9954 R12: 0000000000007fca
[ 171.227211] R13: ffff8881066f4a7f R14: ffff8880aa6e3c00 R15: 0000000000000020
[ 171.228460] FS: 00007f8623719740(0000) GS:ffff88810be00000(0000) knlGS:0000000000000000
[ 171.230065] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 171.231303] CR2: 00007ffdab190a50 CR3: 00000001038ea006 CR4: 0000000000160ee0
[ 171.232586] Call Trace:
[ 171.233109] <IRQ>
[ 171.233531] get_mapping_status (linux-mptcp/net/mptcp/subflow.c:691)
[ 171.234371] mptcp_subflow_data_available (linux-mptcp/net/mptcp/subflow.c:736 linux-mptcp/net/mptcp/subflow.c:832)
[ 171.238181] subflow_state_change (linux-mptcp/net/mptcp/subflow.c:1085 (discriminator 1))
[ 171.239066] tcp_fin (linux-mptcp/net/ipv4/tcp_input.c:4217)
[ 171.240123] tcp_data_queue (linux-mptcp/./include/linux/compiler.h:199 linux-mptcp/net/ipv4/tcp_input.c:4822)
[ 171.245083] tcp_rcv_established (linux-mptcp/./include/linux/skbuff.h:1785 linux-mptcp/./include/net/tcp.h:1774 linux-mptcp/./include/net/tcp.h:1847 linux-mptcp/net/ipv4/tcp_input.c:5238 linux-mptcp/net/ipv4/tcp_input.c:5730)
[ 171.254089] tcp_v4_rcv (linux-mptcp/./include/linux/spinlock.h:393 linux-mptcp/net/ipv4/tcp_ipv4.c:2009)
[ 171.258969] ip_protocol_deliver_rcu (linux-mptcp/net/ipv4/ip_input.c:204 (discriminator 1))
[ 171.260214] ip_local_deliver_finish (linux-mptcp/./include/linux/rcupdate.h:651 linux-mptcp/net/ipv4/ip_input.c:232)
[ 171.261389] ip_local_deliver (linux-mptcp/./include/linux/netfilter.h:307 linux-mptcp/./include/linux/netfilter.h:301 linux-mptcp/net/ipv4/ip_input.c:252)
[ 171.265884] ip_rcv (linux-mptcp/./include/linux/netfilter.h:307 linux-mptcp/./include/linux/netfilter.h:301 linux-mptcp/net/ipv4/ip_input.c:539)
[ 171.273666] process_backlog (linux-mptcp/./include/linux/rcupdate.h:651 linux-mptcp/net/core/dev.c:6135)
[ 171.275328] net_rx_action (linux-mptcp/net/core/dev.c:6572 linux-mptcp/net/core/dev.c:6640)
[ 171.280472] __do_softirq (linux-mptcp/./arch/x86/include/asm/jump_label.h:25 linux-mptcp/./include/linux/jump_label.h:200 linux-mptcp/./include/trace/events/irq.h:142 linux-mptcp/kernel/softirq.c:293)
[ 171.281379] do_softirq_own_stack (linux-mptcp/arch/x86/entry/entry_64.S:1083)
[ 171.282358] </IRQ>
We could address the issue clearing explicitly the relevant fields
in several places - tcp_parse_option, tcp_fast_parse_options,
possibly others.
Instead we move the MPTCP option parsing into the already existing
mptcp ingress hook, so that we need to clear the fields in a single
place.
This allows us dropping an MPTCP hook from the TCP code and
removing the quite large mptcp_options_received from the tcp_sock
struct. On the flip side, the MPTCP sockets will traverse the
option space twice (in tcp_parse_option() and in
mptcp_incoming_options(). That looks acceptable: we already
do that for syn and 3rd ack packets, plain TCP socket will
benefit from it, and even MPTCP sockets will experience better
code locality, reducing the jumps between TCP and MPTCP code.
v1 -> v2:
- rebased on current '-net' tree
Fixes: 648ef4b88673 ("mptcp: Implement MPTCP receive path")
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-04-30 21:01:52 +08:00
|
|
|
msk->remote_key = mp_opt->sndr_key;
|
2020-03-13 23:52:41 +08:00
|
|
|
mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
|
|
|
|
ack_seq++;
|
2020-09-30 06:08:19 +08:00
|
|
|
WRITE_ONCE(msk->ack_seq, ack_seq);
|
2020-03-13 23:52:41 +08:00
|
|
|
}
|
2020-03-17 22:53:34 +08:00
|
|
|
|
2020-04-20 22:25:04 +08:00
|
|
|
sock_reset_flag(nsk, SOCK_RCU_FREE);
|
2020-03-17 22:53:34 +08:00
|
|
|
/* will be fully established after successful MPC subflow creation */
|
|
|
|
inet_sk_state_store(nsk, TCP_SYN_RECV);
|
2020-03-13 23:52:41 +08:00
|
|
|
bh_unlock_sock(nsk);
|
|
|
|
|
|
|
|
/* keep a single reference */
|
|
|
|
__sock_put(nsk);
|
2020-02-06 07:39:37 +08:00
|
|
|
return nsk;
|
|
|
|
}
|
|
|
|
|
2020-07-01 03:24:45 +08:00
|
|
|
void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)
|
|
|
|
{
|
|
|
|
const struct tcp_sock *tp = tcp_sk(ssk);
|
|
|
|
|
|
|
|
msk->rcvq_space.copied = 0;
|
|
|
|
msk->rcvq_space.rtt_us = 0;
|
|
|
|
|
|
|
|
msk->rcvq_space.time = tp->tcp_mstamp;
|
|
|
|
|
|
|
|
/* initial rcv_space offering made to peer */
|
|
|
|
msk->rcvq_space.space = min_t(u32, tp->rcv_wnd,
|
|
|
|
TCP_INIT_CWND * tp->advmss);
|
|
|
|
if (msk->rcvq_space.space == 0)
|
|
|
|
msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:19 +08:00
|
|
|
static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
|
|
|
|
bool kern)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
struct socket *listener;
|
|
|
|
struct sock *newsk;
|
|
|
|
|
|
|
|
listener = __mptcp_nmpc_socket(msk);
|
|
|
|
if (WARN_ON_ONCE(!listener)) {
|
|
|
|
*err = -EINVAL;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
|
|
|
|
newsk = inet_csk_accept(listener->sk, flags, err, kern);
|
|
|
|
if (!newsk)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));
|
|
|
|
if (sk_is_mptcp(newsk)) {
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct sock *new_mptcp_sock;
|
|
|
|
struct sock *ssk = newsk;
|
|
|
|
|
|
|
|
subflow = mptcp_subflow_ctx(newsk);
|
2020-03-13 23:52:41 +08:00
|
|
|
new_mptcp_sock = subflow->conn;
|
2020-01-22 08:56:19 +08:00
|
|
|
|
2020-03-13 23:52:41 +08:00
|
|
|
/* is_mptcp should be false if subflow->conn is missing, see
|
|
|
|
* subflow_syn_recv_sock()
|
|
|
|
*/
|
|
|
|
if (WARN_ON_ONCE(!new_mptcp_sock)) {
|
|
|
|
tcp_sk(newsk)->is_mptcp = 0;
|
|
|
|
return newsk;
|
2020-01-22 08:56:19 +08:00
|
|
|
}
|
|
|
|
|
2020-03-13 23:52:41 +08:00
|
|
|
/* acquire the 2nd reference for the owning socket */
|
|
|
|
sock_hold(new_mptcp_sock);
|
2020-01-22 08:56:19 +08:00
|
|
|
|
2020-03-13 23:52:41 +08:00
|
|
|
local_bh_disable();
|
|
|
|
bh_lock_sock(new_mptcp_sock);
|
2020-01-22 08:56:19 +08:00
|
|
|
msk = mptcp_sk(new_mptcp_sock);
|
mptcp: cope with later TCP fallback
With MPTCP v1, passive connections can fallback to TCP after the
subflow becomes established:
syn + MP_CAPABLE ->
<- syn, ack + MP_CAPABLE
ack, seq = 3 ->
// OoO packet is accepted because in-sequence
// passive socket is created, is in ESTABLISHED
// status and tentatively as MP_CAPABLE
ack, seq = 2 ->
// no MP_CAPABLE opt, subflow should fallback to TCP
We can't use the 'subflow' socket fallback, as we don't have
it available for passive connection.
Instead, when the fallback is detected, replace the mptcp
socket with the underlying TCP subflow. Beyond covering
the above scenario, it makes a TCP fallback socket as efficient
as plain TCP ones.
Co-developed-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-22 08:56:33 +08:00
|
|
|
msk->first = newsk;
|
2020-01-22 08:56:19 +08:00
|
|
|
|
|
|
|
newsk = new_mptcp_sock;
|
|
|
|
mptcp_copy_inaddrs(newsk, ssk);
|
|
|
|
list_add(&subflow->node, &msk->conn_list);
|
|
|
|
|
2020-07-01 03:24:45 +08:00
|
|
|
mptcp_rcv_space_init(msk, ssk);
|
2020-01-22 08:56:19 +08:00
|
|
|
bh_unlock_sock(new_mptcp_sock);
|
2020-03-28 05:48:50 +08:00
|
|
|
|
|
|
|
__MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK);
|
2020-01-22 08:56:19 +08:00
|
|
|
local_bh_enable();
|
2020-03-28 05:48:50 +08:00
|
|
|
} else {
|
|
|
|
MPTCP_INC_STATS(sock_net(sk),
|
|
|
|
MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK);
|
2020-01-22 08:56:19 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return newsk;
|
|
|
|
}
|
|
|
|
|
2020-09-24 08:29:57 +08:00
|
|
|
void mptcp_destroy_common(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
skb_rbtree_purge(&msk->out_of_order_queue);
|
|
|
|
mptcp_token_destroy(msk);
|
|
|
|
mptcp_pm_free_anno_list(msk);
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:20 +08:00
|
|
|
static void mptcp_destroy(struct sock *sk)
|
|
|
|
{
|
2020-01-29 22:54:43 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
if (msk->cached_ext)
|
|
|
|
__skb_ext_put(msk->cached_ext);
|
2020-03-28 05:48:45 +08:00
|
|
|
|
2020-09-24 08:29:57 +08:00
|
|
|
mptcp_destroy_common(msk);
|
2020-03-28 05:48:45 +08:00
|
|
|
sk_sockets_allocated_dec(sk);
|
2020-01-22 08:56:20 +08:00
|
|
|
}
|
|
|
|
|
2020-07-05 07:30:16 +08:00
|
|
|
static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname,
|
2020-07-23 14:09:07 +08:00
|
|
|
sockptr_t optval, unsigned int optlen)
|
2020-07-05 07:30:16 +08:00
|
|
|
{
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
struct socket *ssock;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
switch (optname) {
|
|
|
|
case SO_REUSEPORT:
|
|
|
|
case SO_REUSEADDR:
|
|
|
|
lock_sock(sk);
|
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
if (!ssock) {
|
|
|
|
release_sock(sk);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2020-07-23 14:09:07 +08:00
|
|
|
ret = sock_setsockopt(ssock, SOL_SOCKET, optname, optval, optlen);
|
2020-07-05 07:30:16 +08:00
|
|
|
if (ret == 0) {
|
|
|
|
if (optname == SO_REUSEPORT)
|
|
|
|
sk->sk_reuseport = ssock->sk->sk_reuseport;
|
|
|
|
else if (optname == SO_REUSEADDR)
|
|
|
|
sk->sk_reuse = ssock->sk->sk_reuse;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2020-07-23 14:09:07 +08:00
|
|
|
return sock_setsockopt(sk->sk_socket, SOL_SOCKET, optname, optval, optlen);
|
2020-07-05 07:30:16 +08:00
|
|
|
}
|
|
|
|
|
2020-07-05 07:30:17 +08:00
|
|
|
static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname,
|
2020-07-23 14:09:07 +08:00
|
|
|
sockptr_t optval, unsigned int optlen)
|
2020-07-05 07:30:17 +08:00
|
|
|
{
|
|
|
|
struct sock *sk = (struct sock *)msk;
|
|
|
|
int ret = -EOPNOTSUPP;
|
|
|
|
struct socket *ssock;
|
|
|
|
|
|
|
|
switch (optname) {
|
|
|
|
case IPV6_V6ONLY:
|
|
|
|
lock_sock(sk);
|
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
if (!ssock) {
|
|
|
|
release_sock(sk);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen);
|
|
|
|
if (ret == 0)
|
|
|
|
sk->sk_ipv6only = ssock->sk->sk_ipv6only;
|
|
|
|
|
|
|
|
release_sock(sk);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:22 +08:00
|
|
|
static int mptcp_setsockopt(struct sock *sk, int level, int optname,
|
2020-07-23 14:09:07 +08:00
|
|
|
sockptr_t optval, unsigned int optlen)
|
2020-01-22 08:56:22 +08:00
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
2020-06-30 04:26:24 +08:00
|
|
|
struct sock *ssk;
|
2020-01-22 08:56:22 +08:00
|
|
|
|
|
|
|
pr_debug("msk=%p", msk);
|
|
|
|
|
2020-07-05 07:30:15 +08:00
|
|
|
if (level == SOL_SOCKET)
|
2020-07-05 07:30:16 +08:00
|
|
|
return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen);
|
2020-07-05 07:30:15 +08:00
|
|
|
|
2020-01-22 08:56:22 +08:00
|
|
|
/* @@ the meaning of setsockopt() when the socket is connected and
|
2020-02-15 06:14:29 +08:00
|
|
|
* there are multiple subflows is not yet defined. It is up to the
|
|
|
|
* MPTCP-level socket to configure the subflows until the subflow
|
|
|
|
* is in TCP fallback, when TCP socket options are passed through
|
|
|
|
* to the one remaining subflow.
|
2020-01-22 08:56:22 +08:00
|
|
|
*/
|
|
|
|
lock_sock(sk);
|
2020-06-30 04:26:24 +08:00
|
|
|
ssk = __mptcp_tcp_fallback(msk);
|
2020-04-12 03:05:01 +08:00
|
|
|
release_sock(sk);
|
2020-06-30 04:26:24 +08:00
|
|
|
if (ssk)
|
|
|
|
return tcp_setsockopt(ssk, level, optname, optval, optlen);
|
mptcp: fix panic on user pointer access
Its not possible to call the kernel_(s|g)etsockopt functions here,
the address points to user memory:
General protection fault in user access. Non-canonical address?
WARNING: CPU: 1 PID: 5352 at arch/x86/mm/extable.c:77 ex_handler_uaccess+0xba/0xe0 arch/x86/mm/extable.c:77
Kernel panic - not syncing: panic_on_warn set ...
[..]
Call Trace:
fixup_exception+0x9d/0xcd arch/x86/mm/extable.c:178
general_protection+0x2d/0x40 arch/x86/entry/entry_64.S:1202
do_ip_getsockopt+0x1f6/0x1860 net/ipv4/ip_sockglue.c:1323
ip_getsockopt+0x87/0x1c0 net/ipv4/ip_sockglue.c:1561
tcp_getsockopt net/ipv4/tcp.c:3691 [inline]
tcp_getsockopt+0x8c/0xd0 net/ipv4/tcp.c:3685
kernel_getsockopt+0x121/0x1f0 net/socket.c:3736
mptcp_getsockopt+0x69/0x90 net/mptcp/protocol.c:830
__sys_getsockopt+0x13a/0x220 net/socket.c:2175
We can call tcp_get/setsockopt functions instead. Doing so fixes
crashing, but still leaves rtnl related lockdep splat:
WARNING: possible circular locking dependency detected
5.5.0-rc6 #2 Not tainted
------------------------------------------------------
syz-executor.0/16334 is trying to acquire lock:
ffffffff84f7a080 (rtnl_mutex){+.+.}, at: do_ip_setsockopt.isra.0+0x277/0x3820 net/ipv4/ip_sockglue.c:644
but task is already holding lock:
ffff888116503b90 (sk_lock-AF_INET){+.+.}, at: lock_sock include/net/sock.h:1516 [inline]
ffff888116503b90 (sk_lock-AF_INET){+.+.}, at: mptcp_setsockopt+0x28/0x90 net/mptcp/protocol.c:1284
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (sk_lock-AF_INET){+.+.}:
lock_sock_nested+0xca/0x120 net/core/sock.c:2944
lock_sock include/net/sock.h:1516 [inline]
do_ip_setsockopt.isra.0+0x281/0x3820 net/ipv4/ip_sockglue.c:645
ip_setsockopt+0x44/0xf0 net/ipv4/ip_sockglue.c:1248
udp_setsockopt+0x5d/0xa0 net/ipv4/udp.c:2639
__sys_setsockopt+0x152/0x240 net/socket.c:2130
__do_sys_setsockopt net/socket.c:2146 [inline]
__se_sys_setsockopt net/socket.c:2143 [inline]
__x64_sys_setsockopt+0xba/0x150 net/socket.c:2143
do_syscall_64+0xbd/0x5b0 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #0 (rtnl_mutex){+.+.}:
check_prev_add kernel/locking/lockdep.c:2475 [inline]
check_prevs_add kernel/locking/lockdep.c:2580 [inline]
validate_chain kernel/locking/lockdep.c:2970 [inline]
__lock_acquire+0x1fb2/0x4680 kernel/locking/lockdep.c:3954
lock_acquire+0x127/0x330 kernel/locking/lockdep.c:4484
__mutex_lock_common kernel/locking/mutex.c:956 [inline]
__mutex_lock+0x158/0x1340 kernel/locking/mutex.c:1103
do_ip_setsockopt.isra.0+0x277/0x3820 net/ipv4/ip_sockglue.c:644
ip_setsockopt+0x44/0xf0 net/ipv4/ip_sockglue.c:1248
tcp_setsockopt net/ipv4/tcp.c:3159 [inline]
tcp_setsockopt+0x8c/0xd0 net/ipv4/tcp.c:3153
kernel_setsockopt+0x121/0x1f0 net/socket.c:3767
mptcp_setsockopt+0x69/0x90 net/mptcp/protocol.c:1288
__sys_setsockopt+0x152/0x240 net/socket.c:2130
__do_sys_setsockopt net/socket.c:2146 [inline]
__se_sys_setsockopt net/socket.c:2143 [inline]
__x64_sys_setsockopt+0xba/0x150 net/socket.c:2143
do_syscall_64+0xbd/0x5b0 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(sk_lock-AF_INET);
lock(rtnl_mutex);
lock(sk_lock-AF_INET);
lock(rtnl_mutex);
The lockdep complaint is because we hold mptcp socket lock when calling
the sk_prot get/setsockopt handler, and those might need to acquire the
rtnl mutex. Normally, order is:
rtnl_lock(sk) -> lock_sock
Whereas for mptcp the order is
lock_sock(mptcp_sk) rtnl_lock -> lock_sock(subflow_sk)
We can avoid this by releasing the mptcp socket lock early, but, as Paolo
points out, we need to get/put the subflow socket refcount before doing so
to avoid race with concurrent close().
Fixes: 717e79c867ca5 ("mptcp: Add setsockopt()/getsockopt() socket operations")
Reported-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-29 22:54:44 +08:00
|
|
|
|
2020-07-05 07:30:17 +08:00
|
|
|
if (level == SOL_IPV6)
|
|
|
|
return mptcp_setsockopt_v6(msk, optname, optval, optlen);
|
|
|
|
|
2020-02-15 06:14:29 +08:00
|
|
|
return -EOPNOTSUPP;
|
2020-01-22 08:56:22 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int mptcp_getsockopt(struct sock *sk, int level, int optname,
|
mptcp: fix panic on user pointer access
Its not possible to call the kernel_(s|g)etsockopt functions here,
the address points to user memory:
General protection fault in user access. Non-canonical address?
WARNING: CPU: 1 PID: 5352 at arch/x86/mm/extable.c:77 ex_handler_uaccess+0xba/0xe0 arch/x86/mm/extable.c:77
Kernel panic - not syncing: panic_on_warn set ...
[..]
Call Trace:
fixup_exception+0x9d/0xcd arch/x86/mm/extable.c:178
general_protection+0x2d/0x40 arch/x86/entry/entry_64.S:1202
do_ip_getsockopt+0x1f6/0x1860 net/ipv4/ip_sockglue.c:1323
ip_getsockopt+0x87/0x1c0 net/ipv4/ip_sockglue.c:1561
tcp_getsockopt net/ipv4/tcp.c:3691 [inline]
tcp_getsockopt+0x8c/0xd0 net/ipv4/tcp.c:3685
kernel_getsockopt+0x121/0x1f0 net/socket.c:3736
mptcp_getsockopt+0x69/0x90 net/mptcp/protocol.c:830
__sys_getsockopt+0x13a/0x220 net/socket.c:2175
We can call tcp_get/setsockopt functions instead. Doing so fixes
crashing, but still leaves rtnl related lockdep splat:
WARNING: possible circular locking dependency detected
5.5.0-rc6 #2 Not tainted
------------------------------------------------------
syz-executor.0/16334 is trying to acquire lock:
ffffffff84f7a080 (rtnl_mutex){+.+.}, at: do_ip_setsockopt.isra.0+0x277/0x3820 net/ipv4/ip_sockglue.c:644
but task is already holding lock:
ffff888116503b90 (sk_lock-AF_INET){+.+.}, at: lock_sock include/net/sock.h:1516 [inline]
ffff888116503b90 (sk_lock-AF_INET){+.+.}, at: mptcp_setsockopt+0x28/0x90 net/mptcp/protocol.c:1284
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (sk_lock-AF_INET){+.+.}:
lock_sock_nested+0xca/0x120 net/core/sock.c:2944
lock_sock include/net/sock.h:1516 [inline]
do_ip_setsockopt.isra.0+0x281/0x3820 net/ipv4/ip_sockglue.c:645
ip_setsockopt+0x44/0xf0 net/ipv4/ip_sockglue.c:1248
udp_setsockopt+0x5d/0xa0 net/ipv4/udp.c:2639
__sys_setsockopt+0x152/0x240 net/socket.c:2130
__do_sys_setsockopt net/socket.c:2146 [inline]
__se_sys_setsockopt net/socket.c:2143 [inline]
__x64_sys_setsockopt+0xba/0x150 net/socket.c:2143
do_syscall_64+0xbd/0x5b0 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #0 (rtnl_mutex){+.+.}:
check_prev_add kernel/locking/lockdep.c:2475 [inline]
check_prevs_add kernel/locking/lockdep.c:2580 [inline]
validate_chain kernel/locking/lockdep.c:2970 [inline]
__lock_acquire+0x1fb2/0x4680 kernel/locking/lockdep.c:3954
lock_acquire+0x127/0x330 kernel/locking/lockdep.c:4484
__mutex_lock_common kernel/locking/mutex.c:956 [inline]
__mutex_lock+0x158/0x1340 kernel/locking/mutex.c:1103
do_ip_setsockopt.isra.0+0x277/0x3820 net/ipv4/ip_sockglue.c:644
ip_setsockopt+0x44/0xf0 net/ipv4/ip_sockglue.c:1248
tcp_setsockopt net/ipv4/tcp.c:3159 [inline]
tcp_setsockopt+0x8c/0xd0 net/ipv4/tcp.c:3153
kernel_setsockopt+0x121/0x1f0 net/socket.c:3767
mptcp_setsockopt+0x69/0x90 net/mptcp/protocol.c:1288
__sys_setsockopt+0x152/0x240 net/socket.c:2130
__do_sys_setsockopt net/socket.c:2146 [inline]
__se_sys_setsockopt net/socket.c:2143 [inline]
__x64_sys_setsockopt+0xba/0x150 net/socket.c:2143
do_syscall_64+0xbd/0x5b0 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(sk_lock-AF_INET);
lock(rtnl_mutex);
lock(sk_lock-AF_INET);
lock(rtnl_mutex);
The lockdep complaint is because we hold mptcp socket lock when calling
the sk_prot get/setsockopt handler, and those might need to acquire the
rtnl mutex. Normally, order is:
rtnl_lock(sk) -> lock_sock
Whereas for mptcp the order is
lock_sock(mptcp_sk) rtnl_lock -> lock_sock(subflow_sk)
We can avoid this by releasing the mptcp socket lock early, but, as Paolo
points out, we need to get/put the subflow socket refcount before doing so
to avoid race with concurrent close().
Fixes: 717e79c867ca5 ("mptcp: Add setsockopt()/getsockopt() socket operations")
Reported-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-29 22:54:44 +08:00
|
|
|
char __user *optval, int __user *option)
|
2020-01-22 08:56:22 +08:00
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
2020-06-30 04:26:24 +08:00
|
|
|
struct sock *ssk;
|
2020-01-22 08:56:22 +08:00
|
|
|
|
|
|
|
pr_debug("msk=%p", msk);
|
|
|
|
|
2020-02-15 06:14:29 +08:00
|
|
|
/* @@ the meaning of setsockopt() when the socket is connected and
|
|
|
|
* there are multiple subflows is not yet defined. It is up to the
|
|
|
|
* MPTCP-level socket to configure the subflows until the subflow
|
|
|
|
* is in TCP fallback, when socket options are passed through
|
|
|
|
* to the one remaining subflow.
|
2020-01-22 08:56:22 +08:00
|
|
|
*/
|
|
|
|
lock_sock(sk);
|
2020-06-30 04:26:24 +08:00
|
|
|
ssk = __mptcp_tcp_fallback(msk);
|
2020-04-12 03:05:01 +08:00
|
|
|
release_sock(sk);
|
2020-06-30 04:26:24 +08:00
|
|
|
if (ssk)
|
|
|
|
return tcp_getsockopt(ssk, level, optname, optval, option);
|
mptcp: fix panic on user pointer access
Its not possible to call the kernel_(s|g)etsockopt functions here,
the address points to user memory:
General protection fault in user access. Non-canonical address?
WARNING: CPU: 1 PID: 5352 at arch/x86/mm/extable.c:77 ex_handler_uaccess+0xba/0xe0 arch/x86/mm/extable.c:77
Kernel panic - not syncing: panic_on_warn set ...
[..]
Call Trace:
fixup_exception+0x9d/0xcd arch/x86/mm/extable.c:178
general_protection+0x2d/0x40 arch/x86/entry/entry_64.S:1202
do_ip_getsockopt+0x1f6/0x1860 net/ipv4/ip_sockglue.c:1323
ip_getsockopt+0x87/0x1c0 net/ipv4/ip_sockglue.c:1561
tcp_getsockopt net/ipv4/tcp.c:3691 [inline]
tcp_getsockopt+0x8c/0xd0 net/ipv4/tcp.c:3685
kernel_getsockopt+0x121/0x1f0 net/socket.c:3736
mptcp_getsockopt+0x69/0x90 net/mptcp/protocol.c:830
__sys_getsockopt+0x13a/0x220 net/socket.c:2175
We can call tcp_get/setsockopt functions instead. Doing so fixes
crashing, but still leaves rtnl related lockdep splat:
WARNING: possible circular locking dependency detected
5.5.0-rc6 #2 Not tainted
------------------------------------------------------
syz-executor.0/16334 is trying to acquire lock:
ffffffff84f7a080 (rtnl_mutex){+.+.}, at: do_ip_setsockopt.isra.0+0x277/0x3820 net/ipv4/ip_sockglue.c:644
but task is already holding lock:
ffff888116503b90 (sk_lock-AF_INET){+.+.}, at: lock_sock include/net/sock.h:1516 [inline]
ffff888116503b90 (sk_lock-AF_INET){+.+.}, at: mptcp_setsockopt+0x28/0x90 net/mptcp/protocol.c:1284
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (sk_lock-AF_INET){+.+.}:
lock_sock_nested+0xca/0x120 net/core/sock.c:2944
lock_sock include/net/sock.h:1516 [inline]
do_ip_setsockopt.isra.0+0x281/0x3820 net/ipv4/ip_sockglue.c:645
ip_setsockopt+0x44/0xf0 net/ipv4/ip_sockglue.c:1248
udp_setsockopt+0x5d/0xa0 net/ipv4/udp.c:2639
__sys_setsockopt+0x152/0x240 net/socket.c:2130
__do_sys_setsockopt net/socket.c:2146 [inline]
__se_sys_setsockopt net/socket.c:2143 [inline]
__x64_sys_setsockopt+0xba/0x150 net/socket.c:2143
do_syscall_64+0xbd/0x5b0 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
-> #0 (rtnl_mutex){+.+.}:
check_prev_add kernel/locking/lockdep.c:2475 [inline]
check_prevs_add kernel/locking/lockdep.c:2580 [inline]
validate_chain kernel/locking/lockdep.c:2970 [inline]
__lock_acquire+0x1fb2/0x4680 kernel/locking/lockdep.c:3954
lock_acquire+0x127/0x330 kernel/locking/lockdep.c:4484
__mutex_lock_common kernel/locking/mutex.c:956 [inline]
__mutex_lock+0x158/0x1340 kernel/locking/mutex.c:1103
do_ip_setsockopt.isra.0+0x277/0x3820 net/ipv4/ip_sockglue.c:644
ip_setsockopt+0x44/0xf0 net/ipv4/ip_sockglue.c:1248
tcp_setsockopt net/ipv4/tcp.c:3159 [inline]
tcp_setsockopt+0x8c/0xd0 net/ipv4/tcp.c:3153
kernel_setsockopt+0x121/0x1f0 net/socket.c:3767
mptcp_setsockopt+0x69/0x90 net/mptcp/protocol.c:1288
__sys_setsockopt+0x152/0x240 net/socket.c:2130
__do_sys_setsockopt net/socket.c:2146 [inline]
__se_sys_setsockopt net/socket.c:2143 [inline]
__x64_sys_setsockopt+0xba/0x150 net/socket.c:2143
do_syscall_64+0xbd/0x5b0 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(sk_lock-AF_INET);
lock(rtnl_mutex);
lock(sk_lock-AF_INET);
lock(rtnl_mutex);
The lockdep complaint is because we hold mptcp socket lock when calling
the sk_prot get/setsockopt handler, and those might need to acquire the
rtnl mutex. Normally, order is:
rtnl_lock(sk) -> lock_sock
Whereas for mptcp the order is
lock_sock(mptcp_sk) rtnl_lock -> lock_sock(subflow_sk)
We can avoid this by releasing the mptcp socket lock early, but, as Paolo
points out, we need to get/put the subflow socket refcount before doing so
to avoid race with concurrent close().
Fixes: 717e79c867ca5 ("mptcp: Add setsockopt()/getsockopt() socket operations")
Reported-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-29 22:54:44 +08:00
|
|
|
|
2020-02-15 06:14:29 +08:00
|
|
|
return -EOPNOTSUPP;
|
2020-01-22 08:56:22 +08:00
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
#define MPTCP_DEFERRED_ALL (TCPF_DELACK_TIMER_DEFERRED | \
|
|
|
|
TCPF_WRITE_TIMER_DEFERRED)
|
2020-02-26 17:14:52 +08:00
|
|
|
|
|
|
|
/* this is very alike tcp_release_cb() but we must handle differently a
|
|
|
|
* different set of events
|
|
|
|
*/
|
|
|
|
static void mptcp_release_cb(struct sock *sk)
|
|
|
|
{
|
|
|
|
unsigned long flags, nflags;
|
|
|
|
|
|
|
|
do {
|
|
|
|
flags = sk->sk_tsq_flags;
|
|
|
|
if (!(flags & MPTCP_DEFERRED_ALL))
|
|
|
|
return;
|
|
|
|
nflags = flags & ~MPTCP_DEFERRED_ALL;
|
|
|
|
} while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
|
|
|
|
|
2020-03-28 05:48:44 +08:00
|
|
|
sock_release_ownership(sk);
|
|
|
|
|
2020-02-26 17:14:52 +08:00
|
|
|
if (flags & TCPF_DELACK_TIMER_DEFERRED) {
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
struct sock *ssk;
|
|
|
|
|
|
|
|
ssk = mptcp_subflow_recv_lookup(msk);
|
|
|
|
if (!ssk || !schedule_work(&msk->work))
|
|
|
|
__sock_put(sk);
|
|
|
|
}
|
2020-03-28 05:48:44 +08:00
|
|
|
|
|
|
|
if (flags & TCPF_WRITE_TIMER_DEFERRED) {
|
|
|
|
mptcp_retransmit_handler(sk);
|
|
|
|
__sock_put(sk);
|
|
|
|
}
|
2020-02-26 17:14:52 +08:00
|
|
|
}
|
|
|
|
|
2020-06-27 01:30:00 +08:00
|
|
|
static int mptcp_hash(struct sock *sk)
|
|
|
|
{
|
|
|
|
/* should never be called,
|
|
|
|
* we hash the TCP subflows not the master socket
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mptcp_unhash(struct sock *sk)
|
|
|
|
{
|
|
|
|
/* called from sk_common_release(), but nothing to do here */
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
static int mptcp_get_port(struct sock *sk, unsigned short snum)
|
2020-01-22 08:56:15 +08:00
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
2020-01-22 08:56:18 +08:00
|
|
|
struct socket *ssock;
|
2020-01-22 08:56:15 +08:00
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
pr_debug("msk=%p, subflow=%p", msk, ssock);
|
|
|
|
if (WARN_ON_ONCE(!ssock))
|
|
|
|
return -EINVAL;
|
2020-01-22 08:56:15 +08:00
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
return inet_csk_get_port(ssock->sk, snum);
|
|
|
|
}
|
2020-01-22 08:56:15 +08:00
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
void mptcp_finish_connect(struct sock *ssk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
struct mptcp_sock *msk;
|
|
|
|
struct sock *sk;
|
2020-01-22 08:56:23 +08:00
|
|
|
u64 ack_seq;
|
2020-01-22 08:56:15 +08:00
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
subflow = mptcp_subflow_ctx(ssk);
|
|
|
|
sk = subflow->conn;
|
|
|
|
msk = mptcp_sk(sk);
|
|
|
|
|
2020-01-22 08:56:24 +08:00
|
|
|
pr_debug("msk=%p, token=%u", sk, subflow->token);
|
|
|
|
|
2020-01-22 08:56:23 +08:00
|
|
|
mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq);
|
|
|
|
ack_seq++;
|
2020-01-22 08:56:24 +08:00
|
|
|
subflow->map_seq = ack_seq;
|
|
|
|
subflow->map_subflow_seq = 1;
|
2020-01-22 08:56:23 +08:00
|
|
|
|
2020-01-22 08:56:18 +08:00
|
|
|
/* the socket is not connected yet, no msk/subflow ops can access/race
|
|
|
|
* accessing the field below
|
|
|
|
*/
|
|
|
|
WRITE_ONCE(msk->remote_key, subflow->remote_key);
|
|
|
|
WRITE_ONCE(msk->local_key, subflow->local_key);
|
2020-01-22 08:56:23 +08:00
|
|
|
WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
|
|
|
|
WRITE_ONCE(msk->ack_seq, ack_seq);
|
2020-01-22 08:56:32 +08:00
|
|
|
WRITE_ONCE(msk->can_ack, 1);
|
2020-03-28 05:48:42 +08:00
|
|
|
atomic64_set(&msk->snd_una, msk->write_seq);
|
2020-03-28 05:48:38 +08:00
|
|
|
|
|
|
|
mptcp_pm_new_connection(msk, 0);
|
2020-07-01 03:24:45 +08:00
|
|
|
|
|
|
|
mptcp_rcv_space_init(msk, ssk);
|
2020-01-22 08:56:15 +08:00
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:19 +08:00
|
|
|
static void mptcp_sock_graft(struct sock *sk, struct socket *parent)
|
|
|
|
{
|
|
|
|
write_lock_bh(&sk->sk_callback_lock);
|
|
|
|
rcu_assign_pointer(sk->sk_wq, &parent->wq);
|
|
|
|
sk_set_socket(sk, parent);
|
|
|
|
sk->sk_uid = SOCK_INODE(parent)->i_uid;
|
|
|
|
write_unlock_bh(&sk->sk_callback_lock);
|
|
|
|
}
|
|
|
|
|
2020-03-28 05:48:39 +08:00
|
|
|
bool mptcp_finish_join(struct sock *sk)
|
|
|
|
{
|
|
|
|
struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(subflow->conn);
|
|
|
|
struct sock *parent = (void *)msk;
|
|
|
|
struct socket *parent_sock;
|
2020-03-28 05:48:40 +08:00
|
|
|
bool ret;
|
2020-03-28 05:48:39 +08:00
|
|
|
|
|
|
|
pr_debug("msk=%p, subflow=%p", msk, subflow);
|
|
|
|
|
|
|
|
/* mptcp socket already closing? */
|
2020-07-23 19:02:32 +08:00
|
|
|
if (!mptcp_is_fully_established(parent))
|
2020-03-28 05:48:39 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
if (!msk->pm.server_side)
|
|
|
|
return true;
|
|
|
|
|
2020-05-29 23:43:30 +08:00
|
|
|
if (!mptcp_pm_allow_new_subflow(msk))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* active connections are already on conn_list, and we can't acquire
|
|
|
|
* msk lock here.
|
|
|
|
* use the join list lock as synchronization point and double-check
|
|
|
|
* msk status to avoid racing with mptcp_close()
|
|
|
|
*/
|
|
|
|
spin_lock_bh(&msk->join_list_lock);
|
|
|
|
ret = inet_sk_state_load(parent) == TCP_ESTABLISHED;
|
|
|
|
if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node)))
|
|
|
|
list_add_tail(&subflow->node, &msk->join_list);
|
|
|
|
spin_unlock_bh(&msk->join_list_lock);
|
|
|
|
if (!ret)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* attach to msk socket only after we are sure he will deal with us
|
|
|
|
* at close time
|
|
|
|
*/
|
2020-03-28 05:48:39 +08:00
|
|
|
parent_sock = READ_ONCE(parent->sk_socket);
|
|
|
|
if (parent_sock && !sk->sk_socket)
|
|
|
|
mptcp_sock_graft(sk, parent_sock);
|
2020-09-30 06:08:19 +08:00
|
|
|
subflow->map_seq = READ_ONCE(msk->ack_seq);
|
2020-05-29 23:43:30 +08:00
|
|
|
return true;
|
2020-03-28 05:48:39 +08:00
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:25 +08:00
|
|
|
static bool mptcp_memory_free(const struct sock *sk, int wake)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sk);
|
|
|
|
|
|
|
|
return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
static struct proto mptcp_prot = {
|
|
|
|
.name = "MPTCP",
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.init = mptcp_init_sock,
|
2020-03-28 05:48:43 +08:00
|
|
|
.disconnect = mptcp_disconnect,
|
2020-01-22 08:56:15 +08:00
|
|
|
.close = mptcp_close,
|
2020-01-22 08:56:19 +08:00
|
|
|
.accept = mptcp_accept,
|
2020-01-22 08:56:22 +08:00
|
|
|
.setsockopt = mptcp_setsockopt,
|
|
|
|
.getsockopt = mptcp_getsockopt,
|
2020-01-22 08:56:15 +08:00
|
|
|
.shutdown = tcp_shutdown,
|
2020-01-22 08:56:20 +08:00
|
|
|
.destroy = mptcp_destroy,
|
2020-01-22 08:56:15 +08:00
|
|
|
.sendmsg = mptcp_sendmsg,
|
|
|
|
.recvmsg = mptcp_recvmsg,
|
2020-02-26 17:14:52 +08:00
|
|
|
.release_cb = mptcp_release_cb,
|
2020-06-27 01:30:00 +08:00
|
|
|
.hash = mptcp_hash,
|
|
|
|
.unhash = mptcp_unhash,
|
2020-01-22 08:56:18 +08:00
|
|
|
.get_port = mptcp_get_port,
|
2020-03-28 05:48:45 +08:00
|
|
|
.sockets_allocated = &mptcp_sockets_allocated,
|
|
|
|
.memory_allocated = &tcp_memory_allocated,
|
|
|
|
.memory_pressure = &tcp_memory_pressure,
|
2020-01-22 08:56:25 +08:00
|
|
|
.stream_memory_free = mptcp_memory_free,
|
2020-03-28 05:48:45 +08:00
|
|
|
.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
|
2020-11-09 02:49:59 +08:00
|
|
|
.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
|
2020-03-28 05:48:45 +08:00
|
|
|
.sysctl_mem = sysctl_tcp_mem,
|
2020-01-22 08:56:15 +08:00
|
|
|
.obj_size = sizeof(struct mptcp_sock),
|
2020-06-27 01:30:00 +08:00
|
|
|
.slab_flags = SLAB_TYPESAFE_BY_RCU,
|
2020-01-22 08:56:15 +08:00
|
|
|
.no_autobind = true,
|
|
|
|
};
|
|
|
|
|
2020-01-22 08:56:17 +08:00
|
|
|
static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sock->sk);
|
|
|
|
struct socket *ssock;
|
2020-01-22 08:56:19 +08:00
|
|
|
int err;
|
2020-01-22 08:56:17 +08:00
|
|
|
|
|
|
|
lock_sock(sock->sk);
|
2020-06-30 04:26:23 +08:00
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
if (!ssock) {
|
|
|
|
err = -EINVAL;
|
2020-01-22 08:56:17 +08:00
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ssock->ops->bind(ssock, uaddr, addr_len);
|
2020-01-22 08:56:19 +08:00
|
|
|
if (!err)
|
|
|
|
mptcp_copy_inaddrs(sock->sk, ssock->sk);
|
2020-01-22 08:56:17 +08:00
|
|
|
|
|
|
|
unlock:
|
|
|
|
release_sock(sock->sk);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2020-07-23 19:02:31 +08:00
|
|
|
static void mptcp_subflow_early_fallback(struct mptcp_sock *msk,
|
|
|
|
struct mptcp_subflow_context *subflow)
|
|
|
|
{
|
|
|
|
subflow->request_mptcp = 0;
|
|
|
|
__mptcp_do_fallback(msk);
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:17 +08:00
|
|
|
static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
|
|
|
|
int addr_len, int flags)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sock->sk);
|
2020-06-27 01:30:00 +08:00
|
|
|
struct mptcp_subflow_context *subflow;
|
2020-01-22 08:56:17 +08:00
|
|
|
struct socket *ssock;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
lock_sock(sock->sk);
|
2020-05-29 23:43:29 +08:00
|
|
|
if (sock->state != SS_UNCONNECTED && msk->subflow) {
|
|
|
|
/* pending connection or invalid state, let existing subflow
|
|
|
|
* cope with that
|
|
|
|
*/
|
|
|
|
ssock = msk->subflow;
|
|
|
|
goto do_connect;
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:23 +08:00
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
if (!ssock) {
|
|
|
|
err = -EINVAL;
|
2020-01-22 08:56:17 +08:00
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:23 +08:00
|
|
|
mptcp_token_destroy(msk);
|
|
|
|
inet_sk_state_store(sock->sk, TCP_SYN_SENT);
|
2020-06-27 01:30:00 +08:00
|
|
|
subflow = mptcp_subflow_ctx(ssock->sk);
|
2020-01-22 08:56:19 +08:00
|
|
|
#ifdef CONFIG_TCP_MD5SIG
|
|
|
|
/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
|
|
|
|
* TCP option space.
|
|
|
|
*/
|
|
|
|
if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
|
2020-07-23 19:02:31 +08:00
|
|
|
mptcp_subflow_early_fallback(msk, subflow);
|
2020-01-22 08:56:19 +08:00
|
|
|
#endif
|
2020-06-27 01:30:00 +08:00
|
|
|
if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk))
|
2020-07-23 19:02:31 +08:00
|
|
|
mptcp_subflow_early_fallback(msk, subflow);
|
2020-01-22 08:56:19 +08:00
|
|
|
|
2020-05-29 23:43:29 +08:00
|
|
|
do_connect:
|
2020-01-22 08:56:17 +08:00
|
|
|
err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
|
2020-05-29 23:43:29 +08:00
|
|
|
sock->state = ssock->state;
|
|
|
|
|
|
|
|
/* on successful connect, the msk state will be moved to established by
|
|
|
|
* subflow_finish_connect()
|
|
|
|
*/
|
2020-07-27 18:24:33 +08:00
|
|
|
if (!err || err == -EINPROGRESS)
|
2020-05-29 23:43:29 +08:00
|
|
|
mptcp_copy_inaddrs(sock->sk, ssock->sk);
|
|
|
|
else
|
|
|
|
inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
|
2020-01-22 08:56:17 +08:00
|
|
|
|
|
|
|
unlock:
|
|
|
|
release_sock(sock->sk);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:19 +08:00
|
|
|
static int mptcp_listen(struct socket *sock, int backlog)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sock->sk);
|
|
|
|
struct socket *ssock;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
pr_debug("msk=%p", msk);
|
|
|
|
|
|
|
|
lock_sock(sock->sk);
|
2020-06-30 04:26:23 +08:00
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
if (!ssock) {
|
|
|
|
err = -EINVAL;
|
2020-01-22 08:56:19 +08:00
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:23 +08:00
|
|
|
mptcp_token_destroy(msk);
|
|
|
|
inet_sk_state_store(sock->sk, TCP_LISTEN);
|
2020-04-20 22:25:04 +08:00
|
|
|
sock_set_flag(sock->sk, SOCK_RCU_FREE);
|
|
|
|
|
2020-01-22 08:56:19 +08:00
|
|
|
err = ssock->ops->listen(ssock, backlog);
|
|
|
|
inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
|
|
|
|
if (!err)
|
|
|
|
mptcp_copy_inaddrs(sock->sk, ssock->sk);
|
|
|
|
|
|
|
|
unlock:
|
|
|
|
release_sock(sock->sk);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
|
|
|
|
int flags, bool kern)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sock->sk);
|
|
|
|
struct socket *ssock;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
pr_debug("msk=%p", msk);
|
|
|
|
|
|
|
|
lock_sock(sock->sk);
|
|
|
|
if (sock->sk->sk_state != TCP_LISTEN)
|
|
|
|
goto unlock_fail;
|
|
|
|
|
|
|
|
ssock = __mptcp_nmpc_socket(msk);
|
|
|
|
if (!ssock)
|
|
|
|
goto unlock_fail;
|
|
|
|
|
2020-06-30 04:26:25 +08:00
|
|
|
clear_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-01-22 08:56:19 +08:00
|
|
|
sock_hold(ssock->sk);
|
|
|
|
release_sock(sock->sk);
|
|
|
|
|
|
|
|
err = ssock->ops->accept(sock, newsock, flags, kern);
|
2020-06-30 04:26:22 +08:00
|
|
|
if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) {
|
2020-01-22 08:56:19 +08:00
|
|
|
struct mptcp_sock *msk = mptcp_sk(newsock->sk);
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
|
|
|
|
/* set ssk->sk_socket of accept()ed flows to mptcp socket.
|
|
|
|
* This is needed so NOSPACE flag can be set from tcp stack.
|
|
|
|
*/
|
2020-03-28 05:48:40 +08:00
|
|
|
__mptcp_flush_join_list(msk);
|
2020-08-03 21:00:44 +08:00
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
2020-01-22 08:56:19 +08:00
|
|
|
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
|
|
|
if (!ssk->sk_socket)
|
|
|
|
mptcp_sock_graft(ssk, newsock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:25 +08:00
|
|
|
if (inet_csk_listen_poll(ssock->sk))
|
|
|
|
set_bit(MPTCP_DATA_READY, &msk->flags);
|
2020-01-22 08:56:19 +08:00
|
|
|
sock_put(ssock->sk);
|
|
|
|
return err;
|
|
|
|
|
|
|
|
unlock_fail:
|
|
|
|
release_sock(sock->sk);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:25 +08:00
|
|
|
static __poll_t mptcp_check_readable(struct mptcp_sock *msk)
|
|
|
|
{
|
|
|
|
return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM :
|
|
|
|
0;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:17 +08:00
|
|
|
static __poll_t mptcp_poll(struct file *file, struct socket *sock,
|
|
|
|
struct poll_table_struct *wait)
|
|
|
|
{
|
2020-01-22 08:56:25 +08:00
|
|
|
struct sock *sk = sock->sk;
|
mptcp: cope with later TCP fallback
With MPTCP v1, passive connections can fallback to TCP after the
subflow becomes established:
syn + MP_CAPABLE ->
<- syn, ack + MP_CAPABLE
ack, seq = 3 ->
// OoO packet is accepted because in-sequence
// passive socket is created, is in ESTABLISHED
// status and tentatively as MP_CAPABLE
ack, seq = 2 ->
// no MP_CAPABLE opt, subflow should fallback to TCP
We can't use the 'subflow' socket fallback, as we don't have
it available for passive connection.
Instead, when the fallback is detected, replace the mptcp
socket with the underlying TCP subflow. Beyond covering
the above scenario, it makes a TCP fallback socket as efficient
as plain TCP ones.
Co-developed-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Christoph Paasch <cpaasch@apple.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-22 08:56:33 +08:00
|
|
|
struct mptcp_sock *msk;
|
2020-01-22 08:56:17 +08:00
|
|
|
__poll_t mask = 0;
|
2020-06-30 04:26:25 +08:00
|
|
|
int state;
|
2020-01-22 08:56:17 +08:00
|
|
|
|
2020-01-22 08:56:25 +08:00
|
|
|
msk = mptcp_sk(sk);
|
|
|
|
sock_poll_wait(file, sock, wait);
|
|
|
|
|
2020-06-30 04:26:25 +08:00
|
|
|
state = inet_sk_state_load(sk);
|
2020-09-14 16:01:09 +08:00
|
|
|
pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags);
|
2020-06-30 04:26:25 +08:00
|
|
|
if (state == TCP_LISTEN)
|
|
|
|
return mptcp_check_readable(msk);
|
|
|
|
|
|
|
|
if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) {
|
|
|
|
mask |= mptcp_check_readable(msk);
|
2020-09-14 16:01:07 +08:00
|
|
|
if (test_bit(MPTCP_SEND_SPACE, &msk->flags))
|
2020-06-30 04:26:25 +08:00
|
|
|
mask |= EPOLLOUT | EPOLLWRNORM;
|
|
|
|
}
|
2020-01-22 08:56:25 +08:00
|
|
|
if (sk->sk_shutdown & RCV_SHUTDOWN)
|
|
|
|
mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
|
|
|
|
|
2020-01-22 08:56:17 +08:00
|
|
|
return mask;
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:21 +08:00
|
|
|
static int mptcp_shutdown(struct socket *sock, int how)
|
|
|
|
{
|
|
|
|
struct mptcp_sock *msk = mptcp_sk(sock->sk);
|
|
|
|
struct mptcp_subflow_context *subflow;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
pr_debug("sk=%p, how=%d", msk, how);
|
|
|
|
|
|
|
|
lock_sock(sock->sk);
|
|
|
|
|
|
|
|
how++;
|
|
|
|
if ((how & ~SHUTDOWN_MASK) || !how) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sock->state == SS_CONNECTING) {
|
|
|
|
if ((1 << sock->sk->sk_state) &
|
|
|
|
(TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))
|
|
|
|
sock->state = SS_DISCONNECTING;
|
|
|
|
else
|
|
|
|
sock->state = SS_CONNECTED;
|
|
|
|
}
|
|
|
|
|
2020-07-29 06:12:06 +08:00
|
|
|
/* If we've already sent a FIN, or it's a closed state, skip this. */
|
|
|
|
if (__mptcp_check_fallback(msk)) {
|
|
|
|
if (how == SHUT_WR || how == SHUT_RDWR)
|
|
|
|
inet_sk_state_store(sock->sk, TCP_FIN_WAIT1);
|
2020-07-29 06:12:02 +08:00
|
|
|
|
2020-07-29 06:12:06 +08:00
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
|
2020-01-22 08:56:21 +08:00
|
|
|
|
2020-07-29 06:12:06 +08:00
|
|
|
mptcp_subflow_shutdown(sock->sk, tcp_sk, how);
|
|
|
|
}
|
|
|
|
} else if ((how & SEND_SHUTDOWN) &&
|
|
|
|
((1 << sock->sk->sk_state) &
|
|
|
|
(TCPF_ESTABLISHED | TCPF_SYN_SENT |
|
|
|
|
TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) &&
|
|
|
|
mptcp_close_state(sock->sk)) {
|
|
|
|
__mptcp_flush_join_list(msk);
|
|
|
|
|
|
|
|
WRITE_ONCE(msk->write_seq, msk->write_seq + 1);
|
|
|
|
WRITE_ONCE(msk->snd_data_fin_enable, 1);
|
|
|
|
|
|
|
|
mptcp_for_each_subflow(msk, subflow) {
|
|
|
|
struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
|
|
|
|
|
|
|
|
mptcp_subflow_shutdown(sock->sk, tcp_sk, how);
|
|
|
|
}
|
2020-01-22 08:56:21 +08:00
|
|
|
}
|
|
|
|
|
2020-06-30 04:26:20 +08:00
|
|
|
/* Wake up anyone sleeping in poll. */
|
|
|
|
sock->sk->sk_state_change(sock->sk);
|
|
|
|
|
2020-01-22 08:56:21 +08:00
|
|
|
out_unlock:
|
|
|
|
release_sock(sock->sk);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2020-01-25 08:04:02 +08:00
|
|
|
static const struct proto_ops mptcp_stream_ops = {
|
|
|
|
.family = PF_INET,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.release = inet_release,
|
|
|
|
.bind = mptcp_bind,
|
|
|
|
.connect = mptcp_stream_connect,
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
.accept = mptcp_stream_accept,
|
2020-06-30 04:26:22 +08:00
|
|
|
.getname = inet_getname,
|
2020-01-25 08:04:02 +08:00
|
|
|
.poll = mptcp_poll,
|
|
|
|
.ioctl = inet_ioctl,
|
|
|
|
.gettstamp = sock_gettstamp,
|
|
|
|
.listen = mptcp_listen,
|
|
|
|
.shutdown = mptcp_shutdown,
|
|
|
|
.setsockopt = sock_common_setsockopt,
|
|
|
|
.getsockopt = sock_common_getsockopt,
|
|
|
|
.sendmsg = inet_sendmsg,
|
|
|
|
.recvmsg = inet_recvmsg,
|
|
|
|
.mmap = sock_no_mmap,
|
|
|
|
.sendpage = inet_sendpage,
|
|
|
|
};
|
2020-01-22 08:56:17 +08:00
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
static struct inet_protosw mptcp_protosw = {
|
|
|
|
.type = SOCK_STREAM,
|
|
|
|
.protocol = IPPROTO_MPTCP,
|
|
|
|
.prot = &mptcp_prot,
|
2020-01-22 08:56:17 +08:00
|
|
|
.ops = &mptcp_stream_ops,
|
|
|
|
.flags = INET_PROTOSW_ICSK,
|
2020-01-22 08:56:15 +08:00
|
|
|
};
|
|
|
|
|
2020-06-27 01:29:59 +08:00
|
|
|
void __init mptcp_proto_init(void)
|
2020-01-22 08:56:15 +08:00
|
|
|
{
|
2020-01-22 08:56:17 +08:00
|
|
|
mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
|
|
|
|
|
2020-03-28 05:48:45 +08:00
|
|
|
if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL))
|
|
|
|
panic("Failed to allocate MPTCP pcpu counter\n");
|
|
|
|
|
2020-01-22 08:56:17 +08:00
|
|
|
mptcp_subflow_init();
|
2020-03-28 05:48:38 +08:00
|
|
|
mptcp_pm_init();
|
2020-06-27 01:30:00 +08:00
|
|
|
mptcp_token_init();
|
2020-01-22 08:56:17 +08:00
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
if (proto_register(&mptcp_prot, 1) != 0)
|
|
|
|
panic("Failed to register MPTCP proto.\n");
|
|
|
|
|
|
|
|
inet_register_protosw(&mptcp_protosw);
|
2020-02-26 17:14:48 +08:00
|
|
|
|
|
|
|
BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb));
|
2020-01-22 08:56:15 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
|
2020-01-25 08:04:02 +08:00
|
|
|
static const struct proto_ops mptcp_v6_stream_ops = {
|
|
|
|
.family = PF_INET6,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.release = inet6_release,
|
|
|
|
.bind = mptcp_bind,
|
|
|
|
.connect = mptcp_stream_connect,
|
|
|
|
.socketpair = sock_no_socketpair,
|
|
|
|
.accept = mptcp_stream_accept,
|
2020-06-30 04:26:22 +08:00
|
|
|
.getname = inet6_getname,
|
2020-01-25 08:04:02 +08:00
|
|
|
.poll = mptcp_poll,
|
|
|
|
.ioctl = inet6_ioctl,
|
|
|
|
.gettstamp = sock_gettstamp,
|
|
|
|
.listen = mptcp_listen,
|
|
|
|
.shutdown = mptcp_shutdown,
|
|
|
|
.setsockopt = sock_common_setsockopt,
|
|
|
|
.getsockopt = sock_common_getsockopt,
|
|
|
|
.sendmsg = inet6_sendmsg,
|
|
|
|
.recvmsg = inet6_recvmsg,
|
|
|
|
.mmap = sock_no_mmap,
|
|
|
|
.sendpage = inet_sendpage,
|
|
|
|
#ifdef CONFIG_COMPAT
|
2020-05-18 14:28:06 +08:00
|
|
|
.compat_ioctl = inet6_compat_ioctl,
|
2020-01-25 08:04:02 +08:00
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
static struct proto mptcp_v6_prot;
|
|
|
|
|
2020-01-22 08:56:20 +08:00
|
|
|
static void mptcp_v6_destroy(struct sock *sk)
|
|
|
|
{
|
|
|
|
mptcp_destroy(sk);
|
|
|
|
inet6_destroy_sock(sk);
|
|
|
|
}
|
|
|
|
|
2020-01-22 08:56:15 +08:00
|
|
|
static struct inet_protosw mptcp_v6_protosw = {
|
|
|
|
.type = SOCK_STREAM,
|
|
|
|
.protocol = IPPROTO_MPTCP,
|
|
|
|
.prot = &mptcp_v6_prot,
|
2020-01-22 08:56:17 +08:00
|
|
|
.ops = &mptcp_v6_stream_ops,
|
2020-01-22 08:56:15 +08:00
|
|
|
.flags = INET_PROTOSW_ICSK,
|
|
|
|
};
|
|
|
|
|
2020-06-27 01:29:59 +08:00
|
|
|
int __init mptcp_proto_v6_init(void)
|
2020-01-22 08:56:15 +08:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
mptcp_v6_prot = mptcp_prot;
|
|
|
|
strcpy(mptcp_v6_prot.name, "MPTCPv6");
|
|
|
|
mptcp_v6_prot.slab = NULL;
|
2020-01-22 08:56:20 +08:00
|
|
|
mptcp_v6_prot.destroy = mptcp_v6_destroy;
|
2020-02-06 07:39:37 +08:00
|
|
|
mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock);
|
2020-01-22 08:56:15 +08:00
|
|
|
|
|
|
|
err = proto_register(&mptcp_v6_prot, 1);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
err = inet6_register_protosw(&mptcp_v6_protosw);
|
|
|
|
if (err)
|
|
|
|
proto_unregister(&mptcp_v6_prot);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
#endif
|