2019-05-27 14:55:01 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2013-06-08 18:56:03 +08:00
|
|
|
/*
|
|
|
|
* IPV4 GSO/GRO offload support
|
|
|
|
* Linux INET implementation
|
|
|
|
*
|
|
|
|
* UDPv4 GSO support
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/skbuff.h>
|
2021-11-16 01:05:51 +08:00
|
|
|
#include <net/gro.h>
|
2023-06-09 03:17:37 +08:00
|
|
|
#include <net/gso.h>
|
2013-06-08 18:56:03 +08:00
|
|
|
#include <net/udp.h>
|
|
|
|
#include <net/protocol.h>
|
2018-12-14 18:51:59 +08:00
|
|
|
#include <net/inet_common.h>
|
2013-06-08 18:56:03 +08:00
|
|
|
|
2014-09-30 11:22:29 +08:00
|
|
|
static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
|
|
|
|
netdev_features_t features,
|
|
|
|
struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
|
|
|
|
netdev_features_t features),
|
2014-11-05 01:06:52 +08:00
|
|
|
__be16 new_protocol, bool is_ipv6)
|
2014-07-14 10:49:56 +08:00
|
|
|
{
|
2016-02-06 07:28:20 +08:00
|
|
|
int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
|
2017-07-03 22:29:12 +08:00
|
|
|
bool remcsum, need_csum, offload_csum, gso_partial;
|
2014-07-14 10:49:56 +08:00
|
|
|
struct sk_buff *segs = ERR_PTR(-EINVAL);
|
2016-02-06 07:28:20 +08:00
|
|
|
struct udphdr *uh = udp_hdr(skb);
|
2014-07-14 10:49:56 +08:00
|
|
|
u16 mac_offset = skb->mac_header;
|
|
|
|
__be16 protocol = skb->protocol;
|
2016-02-06 07:28:20 +08:00
|
|
|
u16 mac_len = skb->mac_len;
|
2014-07-14 10:49:56 +08:00
|
|
|
int udp_offset, outer_hlen;
|
2016-03-12 06:05:47 +08:00
|
|
|
__wsum partial;
|
2017-04-22 06:23:05 +08:00
|
|
|
bool need_ipsec;
|
2014-07-14 10:49:56 +08:00
|
|
|
|
|
|
|
if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
|
|
|
|
goto out;
|
|
|
|
|
2016-03-12 06:05:47 +08:00
|
|
|
/* Adjust partial header checksum to negate old length.
|
|
|
|
* We cannot rely on the value contained in uh->len as it is
|
|
|
|
* possible that the actual value exceeds the boundaries of the
|
|
|
|
* 16 bit length field due to the header being added outside of an
|
|
|
|
* IP or IPv6 frame that was already limited to 64K - 1.
|
|
|
|
*/
|
2016-04-11 09:45:03 +08:00
|
|
|
if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)
|
|
|
|
partial = (__force __wsum)uh->len;
|
|
|
|
else
|
|
|
|
partial = (__force __wsum)htonl(skb->len);
|
|
|
|
partial = csum_sub(csum_unfold(uh->check), partial);
|
2016-02-06 07:28:20 +08:00
|
|
|
|
|
|
|
/* setup inner skb. */
|
2014-07-14 10:49:56 +08:00
|
|
|
skb->encapsulation = 0;
|
2016-03-23 07:18:07 +08:00
|
|
|
SKB_GSO_CB(skb)->encap_level = 0;
|
2014-07-14 10:49:56 +08:00
|
|
|
__skb_pull(skb, tnl_hlen);
|
|
|
|
skb_reset_mac_header(skb);
|
|
|
|
skb_set_network_header(skb, skb_inner_network_offset(skb));
|
2020-10-29 15:04:57 +08:00
|
|
|
skb_set_transport_header(skb, skb_inner_transport_offset(skb));
|
2014-07-14 10:49:56 +08:00
|
|
|
skb->mac_len = skb_inner_network_offset(skb);
|
2014-09-30 11:22:29 +08:00
|
|
|
skb->protocol = new_protocol;
|
2016-02-06 07:28:14 +08:00
|
|
|
|
|
|
|
need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
|
2014-11-05 01:06:52 +08:00
|
|
|
skb->encap_hdr_csum = need_csum;
|
2016-02-06 07:28:14 +08:00
|
|
|
|
|
|
|
remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);
|
2014-11-05 01:06:54 +08:00
|
|
|
skb->remcsum_offload = remcsum;
|
2014-07-14 10:49:56 +08:00
|
|
|
|
2017-04-22 06:23:05 +08:00
|
|
|
need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));
|
2014-11-05 01:06:52 +08:00
|
|
|
/* Try to offload checksum if possible */
|
|
|
|
offload_csum = !!(need_csum &&
|
2017-04-22 06:23:05 +08:00
|
|
|
!need_ipsec &&
|
2016-02-06 07:28:14 +08:00
|
|
|
(skb->dev->features &
|
|
|
|
(is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) :
|
|
|
|
(NETIF_F_HW_CSUM | NETIF_F_IP_CSUM))));
|
2014-07-14 10:49:56 +08:00
|
|
|
|
2016-02-06 07:27:31 +08:00
|
|
|
features &= skb->dev->hw_enc_features;
|
2021-01-16 13:59:17 +08:00
|
|
|
if (need_csum)
|
|
|
|
features &= ~NETIF_F_SCTP_CRC;
|
2016-02-06 07:27:31 +08:00
|
|
|
|
2016-02-06 07:27:43 +08:00
|
|
|
/* The only checksum offload we care about from here on out is the
|
|
|
|
* outer one so strip the existing checksum feature flags and
|
|
|
|
* instead set the flag based on our outer checksum offload value.
|
|
|
|
*/
|
2017-07-03 22:29:12 +08:00
|
|
|
if (remcsum) {
|
2016-02-06 07:27:43 +08:00
|
|
|
features &= ~NETIF_F_CSUM_MASK;
|
2016-02-25 08:46:21 +08:00
|
|
|
if (!need_csum || offload_csum)
|
2016-02-06 07:27:43 +08:00
|
|
|
features |= NETIF_F_HW_CSUM;
|
|
|
|
}
|
|
|
|
|
2014-07-14 10:49:56 +08:00
|
|
|
/* segment inner packet. */
|
2016-02-06 07:27:31 +08:00
|
|
|
segs = gso_inner_segment(skb, features);
|
2014-07-27 15:08:38 +08:00
|
|
|
if (IS_ERR_OR_NULL(segs)) {
|
2014-07-14 10:49:56 +08:00
|
|
|
skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
|
|
|
|
mac_len);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2016-09-19 18:58:47 +08:00
|
|
|
gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
|
|
|
|
|
2014-07-14 10:49:56 +08:00
|
|
|
outer_hlen = skb_tnl_header_len(skb);
|
|
|
|
udp_offset = outer_hlen - tnl_hlen;
|
|
|
|
skb = segs;
|
|
|
|
do {
|
2016-04-11 09:45:03 +08:00
|
|
|
unsigned int len;
|
2014-11-05 01:06:52 +08:00
|
|
|
|
2016-02-06 07:28:14 +08:00
|
|
|
if (remcsum)
|
2014-11-05 01:06:52 +08:00
|
|
|
skb->ip_summed = CHECKSUM_NONE;
|
2016-02-06 07:28:14 +08:00
|
|
|
|
|
|
|
/* Set up inner headers if we are offloading inner checksum */
|
|
|
|
if (skb->ip_summed == CHECKSUM_PARTIAL) {
|
2014-11-05 01:06:52 +08:00
|
|
|
skb_reset_inner_headers(skb);
|
|
|
|
skb->encapsulation = 1;
|
|
|
|
}
|
2014-07-14 10:49:56 +08:00
|
|
|
|
|
|
|
skb->mac_len = mac_len;
|
2014-11-05 01:06:52 +08:00
|
|
|
skb->protocol = protocol;
|
2014-07-14 10:49:56 +08:00
|
|
|
|
2016-02-06 07:28:20 +08:00
|
|
|
__skb_push(skb, outer_hlen);
|
2014-07-14 10:49:56 +08:00
|
|
|
skb_reset_mac_header(skb);
|
|
|
|
skb_set_network_header(skb, mac_len);
|
|
|
|
skb_set_transport_header(skb, udp_offset);
|
2016-04-11 09:45:03 +08:00
|
|
|
len = skb->len - udp_offset;
|
2014-07-14 10:49:56 +08:00
|
|
|
uh = udp_hdr(skb);
|
2016-04-11 09:45:03 +08:00
|
|
|
|
|
|
|
/* If we are only performing partial GSO the inner header
|
|
|
|
* will be using a length value equal to only one MSS sized
|
|
|
|
* segment instead of the entire frame.
|
|
|
|
*/
|
2017-10-07 00:02:35 +08:00
|
|
|
if (gso_partial && skb_is_gso(skb)) {
|
2016-04-11 09:45:03 +08:00
|
|
|
uh->len = htons(skb_shinfo(skb)->gso_size +
|
|
|
|
SKB_GSO_CB(skb)->data_offset +
|
|
|
|
skb->head - (unsigned char *)uh);
|
|
|
|
} else {
|
|
|
|
uh->len = htons(len);
|
|
|
|
}
|
2014-07-14 10:49:56 +08:00
|
|
|
|
2014-11-05 01:06:52 +08:00
|
|
|
if (!need_csum)
|
|
|
|
continue;
|
|
|
|
|
2016-04-11 09:45:03 +08:00
|
|
|
uh->check = ~csum_fold(csum_add(partial,
|
|
|
|
(__force __wsum)htonl(len)));
|
2014-07-14 10:49:56 +08:00
|
|
|
|
2016-02-06 07:28:14 +08:00
|
|
|
if (skb->encapsulation || !offload_csum) {
|
|
|
|
uh->check = gso_make_checksum(skb, ~uh->check);
|
2014-07-14 10:49:56 +08:00
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
2016-02-06 07:28:14 +08:00
|
|
|
} else {
|
|
|
|
skb->ip_summed = CHECKSUM_PARTIAL;
|
|
|
|
skb->csum_start = skb_transport_header(skb) - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct udphdr, check);
|
2014-07-14 10:49:56 +08:00
|
|
|
}
|
|
|
|
} while ((skb = skb->next));
|
|
|
|
out:
|
|
|
|
return segs;
|
|
|
|
}
|
|
|
|
|
2014-09-30 11:22:29 +08:00
|
|
|
struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
|
|
|
|
netdev_features_t features,
|
|
|
|
bool is_ipv6)
|
|
|
|
{
|
2021-08-31 11:26:08 +08:00
|
|
|
const struct net_offload __rcu **offloads;
|
2014-09-30 11:22:29 +08:00
|
|
|
__be16 protocol = skb->protocol;
|
|
|
|
const struct net_offload *ops;
|
|
|
|
struct sk_buff *segs = ERR_PTR(-EINVAL);
|
|
|
|
struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
|
|
|
|
netdev_features_t features);
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
|
|
|
switch (skb->inner_protocol_type) {
|
|
|
|
case ENCAP_TYPE_ETHER:
|
|
|
|
protocol = skb->inner_protocol;
|
|
|
|
gso_inner_segment = skb_mac_gso_segment;
|
|
|
|
break;
|
|
|
|
case ENCAP_TYPE_IPPROTO:
|
|
|
|
offloads = is_ipv6 ? inet6_offloads : inet_offloads;
|
|
|
|
ops = rcu_dereference(offloads[skb->inner_ipproto]);
|
|
|
|
if (!ops || !ops->callbacks.gso_segment)
|
|
|
|
goto out_unlock;
|
|
|
|
gso_inner_segment = ops->callbacks.gso_segment;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,
|
2014-11-05 01:06:52 +08:00
|
|
|
protocol, is_ipv6);
|
2014-09-30 11:22:29 +08:00
|
|
|
|
|
|
|
out_unlock:
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return segs;
|
|
|
|
}
|
2016-04-05 23:22:51 +08:00
|
|
|
EXPORT_SYMBOL(skb_udp_tunnel_segment);
|
2014-09-30 11:22:29 +08:00
|
|
|
|
2021-01-30 07:13:27 +08:00
|
|
|
static void __udpv4_gso_segment_csum(struct sk_buff *seg,
|
|
|
|
__be32 *oldip, __be32 *newip,
|
|
|
|
__be16 *oldport, __be16 *newport)
|
|
|
|
{
|
|
|
|
struct udphdr *uh;
|
|
|
|
struct iphdr *iph;
|
|
|
|
|
|
|
|
if (*oldip == *newip && *oldport == *newport)
|
|
|
|
return;
|
|
|
|
|
|
|
|
uh = udp_hdr(seg);
|
|
|
|
iph = ip_hdr(seg);
|
|
|
|
|
|
|
|
if (uh->check) {
|
|
|
|
inet_proto_csum_replace4(&uh->check, seg, *oldip, *newip,
|
|
|
|
true);
|
|
|
|
inet_proto_csum_replace2(&uh->check, seg, *oldport, *newport,
|
|
|
|
false);
|
|
|
|
if (!uh->check)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
|
|
|
}
|
|
|
|
*oldport = *newport;
|
|
|
|
|
|
|
|
csum_replace4(&iph->check, *oldip, *newip);
|
|
|
|
*oldip = *newip;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs)
|
|
|
|
{
|
|
|
|
struct sk_buff *seg;
|
|
|
|
struct udphdr *uh, *uh2;
|
|
|
|
struct iphdr *iph, *iph2;
|
|
|
|
|
|
|
|
seg = segs;
|
|
|
|
uh = udp_hdr(seg);
|
|
|
|
iph = ip_hdr(seg);
|
|
|
|
|
|
|
|
if ((udp_hdr(seg)->dest == udp_hdr(seg->next)->dest) &&
|
|
|
|
(udp_hdr(seg)->source == udp_hdr(seg->next)->source) &&
|
|
|
|
(ip_hdr(seg)->daddr == ip_hdr(seg->next)->daddr) &&
|
|
|
|
(ip_hdr(seg)->saddr == ip_hdr(seg->next)->saddr))
|
|
|
|
return segs;
|
|
|
|
|
|
|
|
while ((seg = seg->next)) {
|
|
|
|
uh2 = udp_hdr(seg);
|
|
|
|
iph2 = ip_hdr(seg);
|
|
|
|
|
|
|
|
__udpv4_gso_segment_csum(seg,
|
|
|
|
&iph2->saddr, &iph->saddr,
|
|
|
|
&uh2->source, &uh->source);
|
|
|
|
__udpv4_gso_segment_csum(seg,
|
|
|
|
&iph2->daddr, &iph->daddr,
|
|
|
|
&uh2->dest, &uh->dest);
|
|
|
|
}
|
|
|
|
|
|
|
|
return segs;
|
|
|
|
}
|
|
|
|
|
2020-01-25 18:26:45 +08:00
|
|
|
static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,
|
2021-01-30 07:13:27 +08:00
|
|
|
netdev_features_t features,
|
|
|
|
bool is_ipv6)
|
2020-01-25 18:26:45 +08:00
|
|
|
{
|
|
|
|
unsigned int mss = skb_shinfo(skb)->gso_size;
|
|
|
|
|
|
|
|
skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
|
|
|
|
if (IS_ERR(skb))
|
|
|
|
return skb;
|
|
|
|
|
|
|
|
udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);
|
|
|
|
|
2021-01-30 07:13:27 +08:00
|
|
|
return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);
|
2020-01-25 18:26:45 +08:00
|
|
|
}
|
|
|
|
|
2018-04-27 01:42:16 +08:00
|
|
|
struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
|
2021-01-30 07:13:27 +08:00
|
|
|
netdev_features_t features, bool is_ipv6)
|
2018-04-27 01:42:16 +08:00
|
|
|
{
|
2018-04-27 01:42:18 +08:00
|
|
|
struct sock *sk = gso_skb->sk;
|
|
|
|
unsigned int sum_truesize = 0;
|
2018-04-27 01:42:16 +08:00
|
|
|
struct sk_buff *segs, *seg;
|
|
|
|
struct udphdr *uh;
|
2018-05-08 02:08:28 +08:00
|
|
|
unsigned int mss;
|
2018-05-08 02:08:52 +08:00
|
|
|
bool copy_dtor;
|
2018-05-08 02:08:34 +08:00
|
|
|
__sum16 check;
|
|
|
|
__be16 newlen;
|
2018-04-27 01:42:16 +08:00
|
|
|
|
2018-05-08 02:08:28 +08:00
|
|
|
mss = skb_shinfo(gso_skb)->gso_size;
|
2018-04-27 01:42:16 +08:00
|
|
|
if (gso_skb->len <= sizeof(*uh) + mss)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
2024-07-30 04:10:12 +08:00
|
|
|
if (unlikely(skb_checksum_start(gso_skb) !=
|
2024-08-19 23:06:21 +08:00
|
|
|
skb_transport_header(gso_skb) &&
|
|
|
|
!(skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST)))
|
2024-07-30 04:10:12 +08:00
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
2023-07-14 01:28:00 +08:00
|
|
|
if (skb_gso_ok(gso_skb, features | NETIF_F_GSO_ROBUST)) {
|
|
|
|
/* Packet is from an untrusted source, reset gso_segs. */
|
|
|
|
skb_shinfo(gso_skb)->gso_segs = DIV_ROUND_UP(gso_skb->len - sizeof(*uh),
|
|
|
|
mss);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2024-10-02 01:17:46 +08:00
|
|
|
if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST) {
|
|
|
|
/* Detect modified geometry and pass those to skb_segment. */
|
|
|
|
if (skb_pagelen(gso_skb) - sizeof(*uh) == skb_shinfo(gso_skb)->gso_size)
|
|
|
|
return __udp_gso_segment_list(gso_skb, features, is_ipv6);
|
|
|
|
|
|
|
|
/* Setup csum, as fraglist skips this in udp4_gro_receive. */
|
|
|
|
gso_skb->csum_start = skb_transport_header(gso_skb) - gso_skb->head;
|
|
|
|
gso_skb->csum_offset = offsetof(struct udphdr, check);
|
|
|
|
gso_skb->ip_summed = CHECKSUM_PARTIAL;
|
|
|
|
|
|
|
|
uh = udp_hdr(gso_skb);
|
|
|
|
if (is_ipv6)
|
|
|
|
uh->check = ~udp_v6_check(gso_skb->len,
|
|
|
|
&ipv6_hdr(gso_skb)->saddr,
|
|
|
|
&ipv6_hdr(gso_skb)->daddr, 0);
|
|
|
|
else
|
|
|
|
uh->check = ~udp_v4_check(gso_skb->len,
|
|
|
|
ip_hdr(gso_skb)->saddr,
|
|
|
|
ip_hdr(gso_skb)->daddr, 0);
|
|
|
|
}
|
2023-07-14 01:28:00 +08:00
|
|
|
|
2018-04-27 01:42:16 +08:00
|
|
|
skb_pull(gso_skb, sizeof(*uh));
|
|
|
|
|
2018-04-27 01:42:18 +08:00
|
|
|
/* clear destructor to avoid skb_segment assigning it to tail */
|
2018-05-08 02:08:52 +08:00
|
|
|
copy_dtor = gso_skb->destructor == sock_wfree;
|
|
|
|
if (copy_dtor)
|
|
|
|
gso_skb->destructor = NULL;
|
2018-04-27 01:42:18 +08:00
|
|
|
|
2018-04-27 01:42:16 +08:00
|
|
|
segs = skb_segment(gso_skb, features);
|
2019-06-06 05:09:05 +08:00
|
|
|
if (IS_ERR_OR_NULL(segs)) {
|
2018-05-08 02:08:52 +08:00
|
|
|
if (copy_dtor)
|
|
|
|
gso_skb->destructor = sock_wfree;
|
2018-04-27 01:42:16 +08:00
|
|
|
return segs;
|
2018-04-27 01:42:18 +08:00
|
|
|
}
|
2018-04-27 01:42:16 +08:00
|
|
|
|
2018-05-08 02:08:46 +08:00
|
|
|
/* GSO partial and frag_list segmentation only requires splitting
|
|
|
|
* the frame into an MSS multiple and possibly a remainder, both
|
|
|
|
* cases return a GSO skb. So update the mss now.
|
|
|
|
*/
|
|
|
|
if (skb_is_gso(segs))
|
|
|
|
mss *= skb_shinfo(segs)->gso_segs;
|
|
|
|
|
2018-05-08 02:08:40 +08:00
|
|
|
seg = segs;
|
|
|
|
uh = udp_hdr(seg);
|
2018-05-08 02:08:34 +08:00
|
|
|
|
2019-06-18 03:05:07 +08:00
|
|
|
/* preserve TX timestamp flags and TS key for first segment */
|
|
|
|
skb_shinfo(seg)->tskey = skb_shinfo(gso_skb)->tskey;
|
|
|
|
skb_shinfo(seg)->tx_flags |=
|
|
|
|
(skb_shinfo(gso_skb)->tx_flags & SKBTX_ANY_TSTAMP);
|
|
|
|
|
2018-05-08 02:08:34 +08:00
|
|
|
/* compute checksum adjustment based on old length versus new */
|
|
|
|
newlen = htons(sizeof(*uh) + mss);
|
|
|
|
check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
|
|
|
|
|
2018-05-08 02:08:40 +08:00
|
|
|
for (;;) {
|
2018-05-08 02:08:52 +08:00
|
|
|
if (copy_dtor) {
|
|
|
|
seg->destructor = sock_wfree;
|
|
|
|
seg->sk = sk;
|
|
|
|
sum_truesize += seg->truesize;
|
|
|
|
}
|
2018-04-27 01:42:16 +08:00
|
|
|
|
2018-05-08 02:08:40 +08:00
|
|
|
if (!seg->next)
|
|
|
|
break;
|
2018-05-08 02:08:34 +08:00
|
|
|
|
|
|
|
uh->len = newlen;
|
|
|
|
uh->check = check;
|
2018-04-27 01:42:18 +08:00
|
|
|
|
2018-05-08 02:08:46 +08:00
|
|
|
if (seg->ip_summed == CHECKSUM_PARTIAL)
|
|
|
|
gso_reset_checksum(seg, ~check);
|
|
|
|
else
|
|
|
|
uh->check = gso_make_checksum(seg, ~check) ? :
|
|
|
|
CSUM_MANGLED_0;
|
|
|
|
|
2018-05-08 02:08:40 +08:00
|
|
|
seg = seg->next;
|
|
|
|
uh = udp_hdr(seg);
|
2018-04-27 01:42:16 +08:00
|
|
|
}
|
|
|
|
|
2018-05-08 02:08:40 +08:00
|
|
|
/* last packet can be partial gso_size, account for that in checksum */
|
|
|
|
newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
|
|
|
|
seg->data_len);
|
|
|
|
check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
|
|
|
|
|
|
|
|
uh->len = newlen;
|
|
|
|
uh->check = check;
|
|
|
|
|
2018-05-08 02:08:46 +08:00
|
|
|
if (seg->ip_summed == CHECKSUM_PARTIAL)
|
|
|
|
gso_reset_checksum(seg, ~check);
|
|
|
|
else
|
|
|
|
uh->check = gso_make_checksum(seg, ~check) ? : CSUM_MANGLED_0;
|
|
|
|
|
2018-05-08 02:08:40 +08:00
|
|
|
/* update refcount for the packet */
|
2018-05-11 10:07:13 +08:00
|
|
|
if (copy_dtor) {
|
|
|
|
int delta = sum_truesize - gso_skb->truesize;
|
|
|
|
|
|
|
|
/* In some pathological cases, delta can be negative.
|
|
|
|
* We need to either use refcount_add() or refcount_sub_and_test()
|
|
|
|
*/
|
|
|
|
if (likely(delta >= 0))
|
|
|
|
refcount_add(delta, &sk->sk_wmem_alloc);
|
|
|
|
else
|
|
|
|
WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc));
|
|
|
|
}
|
2018-04-27 01:42:16 +08:00
|
|
|
return segs;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__udp_gso_segment);
|
|
|
|
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
|
|
|
|
netdev_features_t features)
|
2013-06-08 18:56:03 +08:00
|
|
|
{
|
|
|
|
struct sk_buff *segs = ERR_PTR(-EINVAL);
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
unsigned int mss;
|
|
|
|
__wsum csum;
|
|
|
|
struct udphdr *uh;
|
|
|
|
struct iphdr *iph;
|
2013-12-27 05:10:22 +08:00
|
|
|
|
|
|
|
if (skb->encapsulation &&
|
2014-06-05 08:20:16 +08:00
|
|
|
(skb_shinfo(skb)->gso_type &
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
(SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) {
|
2014-09-30 11:22:29 +08:00
|
|
|
segs = skb_udp_tunnel_segment(skb, features, false);
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2018-04-27 01:42:16 +08:00
|
|
|
if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
|
2018-01-19 22:29:18 +08:00
|
|
|
goto out;
|
|
|
|
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
if (!pskb_may_pull(skb, sizeof(struct udphdr)))
|
|
|
|
goto out;
|
|
|
|
|
2023-07-14 01:28:00 +08:00
|
|
|
if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
|
2021-01-30 07:13:27 +08:00
|
|
|
return __udp_gso_segment(skb, features, false);
|
2018-04-27 01:42:16 +08:00
|
|
|
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
mss = skb_shinfo(skb)->gso_size;
|
|
|
|
if (unlikely(skb->len <= mss))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/* Do software UFO. Complete and fill in the UDP checksum as
|
|
|
|
* HW cannot do checksum of UDP packets sent as multiple
|
|
|
|
* IP fragments.
|
|
|
|
*/
|
2014-09-21 05:52:29 +08:00
|
|
|
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
uh = udp_hdr(skb);
|
|
|
|
iph = ip_hdr(skb);
|
|
|
|
|
|
|
|
uh->check = 0;
|
|
|
|
csum = skb_checksum(skb, 0, skb->len, 0);
|
|
|
|
uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);
|
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
|
|
|
|
|
|
|
skb->ip_summed = CHECKSUM_UNNECESSARY;
|
|
|
|
|
|
|
|
/* If there is no outer header we can fake a checksum offload
|
|
|
|
* due to the fact that we have already done the checksum in
|
|
|
|
* software prior to segmenting the frame.
|
|
|
|
*/
|
|
|
|
if (!skb->encap_hdr_csum)
|
|
|
|
features |= NETIF_F_HW_CSUM;
|
|
|
|
|
|
|
|
/* Fragment the skb. IP headers of the fragments are updated in
|
|
|
|
* inet_gso_segment()
|
|
|
|
*/
|
|
|
|
segs = skb_segment(skb, features);
|
|
|
|
out:
|
2013-06-08 18:56:03 +08:00
|
|
|
return segs;
|
|
|
|
}
|
|
|
|
|
2021-11-16 01:05:52 +08:00
|
|
|
static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
if (unlikely(p->len + skb->len >= 65536))
|
|
|
|
return -E2BIG;
|
|
|
|
|
|
|
|
if (NAPI_GRO_CB(p)->last == p)
|
|
|
|
skb_shinfo(p)->frag_list = skb;
|
|
|
|
else
|
|
|
|
NAPI_GRO_CB(p)->last->next = skb;
|
|
|
|
|
|
|
|
skb_pull(skb, skb_gro_offset(skb));
|
|
|
|
|
|
|
|
NAPI_GRO_CB(p)->last = skb;
|
|
|
|
NAPI_GRO_CB(p)->count++;
|
|
|
|
p->data_len += skb->len;
|
|
|
|
|
2024-03-26 19:33:59 +08:00
|
|
|
/* sk ownership - if any - completely transferred to the aggregated packet */
|
2021-11-16 01:05:52 +08:00
|
|
|
skb->destructor = NULL;
|
2024-03-26 19:33:59 +08:00
|
|
|
skb->sk = NULL;
|
2021-11-16 01:05:52 +08:00
|
|
|
p->truesize += skb->truesize;
|
|
|
|
p->len += skb->len;
|
|
|
|
|
|
|
|
NAPI_GRO_CB(skb)->same_flow = 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-11-07 19:38:29 +08:00
|
|
|
#define UDP_GRO_CNT_MAX 64
|
|
|
|
static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
|
|
|
|
struct sk_buff *skb)
|
|
|
|
{
|
2020-11-12 04:45:25 +08:00
|
|
|
struct udphdr *uh = udp_gro_udphdr(skb);
|
2018-11-07 19:38:29 +08:00
|
|
|
struct sk_buff *pp = NULL;
|
|
|
|
struct udphdr *uh2;
|
|
|
|
struct sk_buff *p;
|
2019-05-02 09:56:28 +08:00
|
|
|
unsigned int ulen;
|
2020-01-25 18:26:45 +08:00
|
|
|
int ret = 0;
|
2024-04-30 22:35:55 +08:00
|
|
|
int flush;
|
2018-11-07 19:38:29 +08:00
|
|
|
|
|
|
|
/* requires non zero csum, for symmetry with GSO */
|
|
|
|
if (!uh->check) {
|
|
|
|
NAPI_GRO_CB(skb)->flush = 1;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2019-05-02 09:56:28 +08:00
|
|
|
/* Do not deal with padded or malicious packets, sorry ! */
|
|
|
|
ulen = ntohs(uh->len);
|
|
|
|
if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {
|
|
|
|
NAPI_GRO_CB(skb)->flush = 1;
|
|
|
|
return NULL;
|
|
|
|
}
|
2018-11-07 19:38:29 +08:00
|
|
|
/* pull encapsulating udp header */
|
|
|
|
skb_gro_pull(skb, sizeof(struct udphdr));
|
|
|
|
|
|
|
|
list_for_each_entry(p, head, list) {
|
|
|
|
if (!NAPI_GRO_CB(p)->same_flow)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
uh2 = udp_hdr(p);
|
|
|
|
|
|
|
|
/* Match ports only, as csum is always non zero */
|
|
|
|
if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) {
|
|
|
|
NAPI_GRO_CB(p)->same_flow = 0;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2020-01-25 18:26:45 +08:00
|
|
|
if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {
|
|
|
|
NAPI_GRO_CB(skb)->flush = 1;
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2024-04-30 22:35:55 +08:00
|
|
|
flush = NAPI_GRO_CB(p)->flush;
|
|
|
|
|
|
|
|
if (NAPI_GRO_CB(p)->flush_id != 1 ||
|
|
|
|
NAPI_GRO_CB(p)->count != 1 ||
|
|
|
|
!NAPI_GRO_CB(p)->is_atomic)
|
|
|
|
flush |= NAPI_GRO_CB(p)->flush_id;
|
|
|
|
else
|
|
|
|
NAPI_GRO_CB(p)->is_atomic = false;
|
|
|
|
|
2018-11-07 19:38:29 +08:00
|
|
|
/* Terminate the flow on len mismatch or if it grow "too much".
|
|
|
|
* Under small packet flood GRO count could elsewhere grow a lot
|
2019-05-02 09:56:28 +08:00
|
|
|
* leading to excessive truesize values.
|
2019-04-26 18:50:44 +08:00
|
|
|
* On len mismatch merge the first packet shorter than gso_size,
|
|
|
|
* otherwise complete the GRO packet.
|
2018-11-07 19:38:29 +08:00
|
|
|
*/
|
2024-04-30 22:35:55 +08:00
|
|
|
if (ulen > ntohs(uh2->len) || flush) {
|
2020-01-25 18:26:45 +08:00
|
|
|
pp = p;
|
|
|
|
} else {
|
|
|
|
if (NAPI_GRO_CB(skb)->is_flist) {
|
|
|
|
if (!pskb_may_pull(skb, skb_gro_offset(skb))) {
|
|
|
|
NAPI_GRO_CB(skb)->flush = 1;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
if ((skb->ip_summed != p->ip_summed) ||
|
|
|
|
(skb->csum_level != p->csum_level)) {
|
|
|
|
NAPI_GRO_CB(skb)->flush = 1;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
ret = skb_gro_receive_list(p, skb);
|
|
|
|
} else {
|
|
|
|
skb_gro_postpull_rcsum(skb, uh,
|
|
|
|
sizeof(struct udphdr));
|
|
|
|
|
|
|
|
ret = skb_gro_receive(p, skb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ret || ulen != ntohs(uh2->len) ||
|
2018-11-07 19:38:29 +08:00
|
|
|
NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
|
|
|
|
pp = p;
|
|
|
|
|
|
|
|
return pp;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* mismatch, but we never need to flush */
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-06-24 13:13:49 +08:00
|
|
|
struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
|
2020-01-25 18:26:45 +08:00
|
|
|
struct udphdr *uh, struct sock *sk)
|
2014-01-20 19:59:19 +08:00
|
|
|
{
|
2018-06-24 13:13:49 +08:00
|
|
|
struct sk_buff *pp = NULL;
|
|
|
|
struct sk_buff *p;
|
2014-08-23 04:34:44 +08:00
|
|
|
struct udphdr *uh2;
|
|
|
|
unsigned int off = skb_gro_offset(skb);
|
2014-01-20 19:59:19 +08:00
|
|
|
int flush = 1;
|
|
|
|
|
2024-03-26 19:33:58 +08:00
|
|
|
/* We can do L4 aggregation only if the packet can't land in a tunnel
|
|
|
|
* otherwise we could corrupt the inner stream. Detecting such packets
|
|
|
|
* cannot be foolproof and the aggregation might still happen in some
|
|
|
|
* cases. Such packets should be caught in udp_unexpected_gso later.
|
2021-03-30 18:28:50 +08:00
|
|
|
*/
|
2020-03-30 23:31:45 +08:00
|
|
|
NAPI_GRO_CB(skb)->is_flist = 0;
|
2021-03-30 18:28:50 +08:00
|
|
|
if (!sk || !udp_sk(sk)->gro_receive) {
|
2024-03-26 19:34:01 +08:00
|
|
|
/* If the packet was locally encapsulated in a UDP tunnel that
|
|
|
|
* wasn't detected above, do not GRO.
|
|
|
|
*/
|
|
|
|
if (skb->encapsulation)
|
|
|
|
goto out;
|
|
|
|
|
2021-03-30 18:28:50 +08:00
|
|
|
if (skb->dev->features & NETIF_F_GRO_FRAGLIST)
|
2023-09-12 17:17:24 +08:00
|
|
|
NAPI_GRO_CB(skb)->is_flist = sk ? !udp_test_bit(GRO_ENABLED, sk) : 1;
|
2018-11-07 19:38:29 +08:00
|
|
|
|
2021-03-30 18:28:50 +08:00
|
|
|
if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||
|
2023-09-12 17:17:24 +08:00
|
|
|
(sk && udp_test_bit(GRO_ENABLED, sk)) || NAPI_GRO_CB(skb)->is_flist)
|
2021-07-03 06:38:43 +08:00
|
|
|
return call_gro_receive(udp_gro_receive_segment, head, skb);
|
|
|
|
|
|
|
|
/* no GRO, be sure flush the current packet */
|
|
|
|
goto out;
|
2018-11-07 19:38:29 +08:00
|
|
|
}
|
|
|
|
|
2021-03-30 18:28:50 +08:00
|
|
|
if (NAPI_GRO_CB(skb)->encap_mark ||
|
2021-02-27 05:22:48 +08:00
|
|
|
(uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&
|
2014-08-28 12:26:56 +08:00
|
|
|
NAPI_GRO_CB(skb)->csum_cnt == 0 &&
|
2021-03-30 18:28:50 +08:00
|
|
|
!NAPI_GRO_CB(skb)->csum_valid))
|
2020-01-25 18:26:45 +08:00
|
|
|
goto out;
|
2014-01-20 19:59:19 +08:00
|
|
|
|
2016-03-20 00:32:01 +08:00
|
|
|
/* mark that this skb passed once through the tunnel gro layer */
|
|
|
|
NAPI_GRO_CB(skb)->encap_mark = 1;
|
2014-01-20 19:59:19 +08:00
|
|
|
|
|
|
|
flush = 0;
|
|
|
|
|
2018-06-24 13:13:49 +08:00
|
|
|
list_for_each_entry(p, head, list) {
|
2014-01-20 19:59:19 +08:00
|
|
|
if (!NAPI_GRO_CB(p)->same_flow)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
uh2 = (struct udphdr *)(p->data + off);
|
2014-08-23 04:34:44 +08:00
|
|
|
|
|
|
|
/* Match ports and either checksums are either both zero
|
|
|
|
* or nonzero.
|
|
|
|
*/
|
|
|
|
if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
|
|
|
|
(!uh->check ^ !uh2->check)) {
|
2014-01-20 19:59:19 +08:00
|
|
|
NAPI_GRO_CB(p)->same_flow = 0;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
|
2014-06-11 09:54:26 +08:00
|
|
|
skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
|
2016-10-20 21:58:02 +08:00
|
|
|
pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
|
2014-01-20 19:59:19 +08:00
|
|
|
|
2020-01-25 18:26:45 +08:00
|
|
|
out:
|
2018-06-30 23:38:55 +08:00
|
|
|
skb_gro_flush_final(skb, pp, flush);
|
2014-01-20 19:59:19 +08:00
|
|
|
return pp;
|
|
|
|
}
|
2016-04-05 23:22:51 +08:00
|
|
|
EXPORT_SYMBOL(udp_gro_receive);
|
2014-01-20 19:59:19 +08:00
|
|
|
|
2020-11-12 04:45:38 +08:00
|
|
|
static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
|
|
|
|
__be16 dport)
|
|
|
|
{
|
|
|
|
const struct iphdr *iph = skb_gro_network_header(skb);
|
2022-11-15 05:57:56 +08:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2023-07-27 23:33:56 +08:00
|
|
|
int iif, sdif;
|
|
|
|
|
|
|
|
inet_get_iif_sdif(skb, &iif, &sdif);
|
2020-11-12 04:45:38 +08:00
|
|
|
|
2022-11-15 05:57:56 +08:00
|
|
|
return __udp4_lib_lookup(net, iph->saddr, sport,
|
2023-07-27 23:33:56 +08:00
|
|
|
iph->daddr, dport, iif,
|
|
|
|
sdif, net->ipv4.udp_table, NULL);
|
2020-11-12 04:45:38 +08:00
|
|
|
}
|
|
|
|
|
2018-12-14 18:51:59 +08:00
|
|
|
INDIRECT_CALLABLE_SCOPE
|
|
|
|
struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
|
2014-08-23 04:34:44 +08:00
|
|
|
{
|
|
|
|
struct udphdr *uh = udp_gro_udphdr(skb);
|
2020-11-12 04:45:38 +08:00
|
|
|
struct sock *sk = NULL;
|
2020-01-25 18:26:45 +08:00
|
|
|
struct sk_buff *pp;
|
2014-08-23 04:34:44 +08:00
|
|
|
|
2020-01-25 18:26:45 +08:00
|
|
|
if (unlikely(!uh))
|
2014-09-01 06:12:43 +08:00
|
|
|
goto flush;
|
2014-08-23 04:34:44 +08:00
|
|
|
|
2014-09-01 06:12:43 +08:00
|
|
|
/* Don't bother verifying checksum if we're going to flush anyway. */
|
2014-09-11 10:23:18 +08:00
|
|
|
if (NAPI_GRO_CB(skb)->flush)
|
2014-09-01 06:12:43 +08:00
|
|
|
goto skip;
|
|
|
|
|
|
|
|
if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
|
|
|
|
inet_gro_compute_pseudo))
|
|
|
|
goto flush;
|
|
|
|
else if (uh->check)
|
2020-01-03 11:51:00 +08:00
|
|
|
skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
|
2014-09-01 06:12:43 +08:00
|
|
|
inet_gro_compute_pseudo);
|
|
|
|
skip:
|
2014-10-04 06:48:08 +08:00
|
|
|
NAPI_GRO_CB(skb)->is_ipv6 = 0;
|
2020-11-12 04:45:38 +08:00
|
|
|
|
|
|
|
if (static_branch_unlikely(&udp_encap_needed_key))
|
|
|
|
sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);
|
|
|
|
|
2020-01-25 18:26:45 +08:00
|
|
|
pp = udp_gro_receive(head, skb, uh, sk);
|
|
|
|
return pp;
|
2014-09-01 06:12:43 +08:00
|
|
|
|
|
|
|
flush:
|
|
|
|
NAPI_GRO_CB(skb)->flush = 1;
|
|
|
|
return NULL;
|
2014-08-23 04:34:44 +08:00
|
|
|
}
|
|
|
|
|
2018-11-07 19:38:29 +08:00
|
|
|
static int udp_gro_complete_segment(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct udphdr *uh = udp_hdr(skb);
|
|
|
|
|
|
|
|
skb->csum_start = (unsigned char *)uh - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct udphdr, check);
|
|
|
|
skb->ip_summed = CHECKSUM_PARTIAL;
|
|
|
|
|
|
|
|
skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
|
|
|
|
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;
|
net, gro: Set inner transport header offset in tcp/udp GRO hook
GSO expects inner transport header offset to be valid when
skb->encapsulation flag is set. GSO uses this value to calculate the length
of an individual segment of a GSO packet in skb_gso_transport_seglen().
However, tcp/udp gro_complete callbacks don't update the
skb->inner_transport_header when processing an encapsulated TCP/UDP
segment. As a result a GRO skb has ->inner_transport_header set to a value
carried over from earlier skb processing.
This can have mild to tragic consequences. From miscalculating the GSO
segment length to triggering a page fault [1], when trying to read TCP/UDP
header at an address past the skb->data page.
The latter scenario leads to an oops report like so:
BUG: unable to handle page fault for address: ffff9fa7ec00d008
#PF: supervisor read access in kernel mode
#PF: error_code(0x0000) - not-present page
PGD 123f201067 P4D 123f201067 PUD 123f209067 PMD 0
Oops: 0000 [#1] SMP NOPTI
CPU: 44 PID: 0 Comm: swapper/44 Not tainted 5.4.53-cloudflare-2020.7.21 #1
Hardware name: HYVE EDGE-METAL-GEN10/HS-1811DLite1, BIOS V2.15 02/21/2020
RIP: 0010:skb_gso_transport_seglen+0x44/0xa0
Code: c0 41 83 e0 11 f6 87 81 00 00 00 20 74 30 0f b7 87 aa 00 00 00 0f [...]
RSP: 0018:ffffad8640bacbb8 EFLAGS: 00010202
RAX: 000000000000feda RBX: ffff9fcc8d31bc00 RCX: ffff9fa7ec00cffc
RDX: ffff9fa7ebffdec0 RSI: 000000000000feda RDI: 0000000000000122
RBP: 00000000000005c4 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9fe588ae3800 R11: ffff9fe011fc92f0 R12: ffff9fcc8d31bc00
R13: ffff9fe0119d4300 R14: 00000000000005c4 R15: ffff9fba57d70900
FS: 0000000000000000(0000) GS:ffff9fe68df00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: ffff9fa7ec00d008 CR3: 0000003e99b1c000 CR4: 0000000000340ee0
Call Trace:
<IRQ>
skb_gso_validate_network_len+0x11/0x70
__ip_finish_output+0x109/0x1c0
ip_sublist_rcv_finish+0x57/0x70
ip_sublist_rcv+0x2aa/0x2d0
? ip_rcv_finish_core.constprop.0+0x390/0x390
ip_list_rcv+0x12b/0x14f
__netif_receive_skb_list_core+0x2a9/0x2d0
netif_receive_skb_list_internal+0x1b5/0x2e0
napi_complete_done+0x93/0x140
veth_poll+0xc0/0x19f [veth]
? mlx5e_napi_poll+0x221/0x610 [mlx5_core]
net_rx_action+0x1f8/0x790
__do_softirq+0xe1/0x2bf
irq_exit+0x8e/0xc0
do_IRQ+0x58/0xe0
common_interrupt+0xf/0xf
</IRQ>
The bug can be observed in a simple setup where we send IP/GRE/IP/TCP
packets into a netns over a veth pair. Inside the netns, packets are
forwarded to dummy device:
trafgen -> [veth A]--[veth B] -forward-> [dummy]
For veth B to GRO aggregate packets on receive, it needs to have an XDP
program attached (for example, a trivial XDP_PASS). Additionally, for UDP,
we need to enable GSO_UDP_L4 feature on the device:
ip netns exec A ethtool -K AB rx-udp-gro-forwarding on
The last component is an artificial delay to increase the chances of GRO
batching happening:
ip netns exec A tc qdisc add dev AB root \
netem delay 200us slot 5ms 10ms packets 2 bytes 64k
With such a setup in place, the bug can be observed by tracing the skb
outer and inner offsets when GSO skb is transmitted from the dummy device:
tcp:
FUNC DEV SKB_LEN NH TH ENC INH ITH GSO_SIZE GSO_TYPE
ip_finish_output dumB 2830 270 290 1 294 254 1383 (tcpv4,gre,)
^^^
udp:
FUNC DEV SKB_LEN NH TH ENC INH ITH GSO_SIZE GSO_TYPE
ip_finish_output dumB 2818 270 290 1 294 254 1383 (gre,udp_l4,)
^^^
Fix it by updating the inner transport header offset in tcp/udp
gro_complete callbacks, similar to how {inet,ipv6}_gro_complete callbacks
update the inner network header offset, when skb->encapsulation flag is
set.
[1] https://lore.kernel.org/netdev/CAKxSbF01cLpZem2GFaUaifh0S-5WYViZemTicAg7FCHOnh6kug@mail.gmail.com/
Fixes: bf296b125b21 ("tcp: Add GRO support")
Fixes: f993bc25e519 ("net: core: handle encapsulation offloads when computing segment lengths")
Fixes: e20cf8d3f1f7 ("udp: implement GRO for plain UDP sockets.")
Reported-by: Alex Forster <aforster@cloudflare.com>
Signed-off-by: Jakub Sitnicki <jakub@cloudflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-07-29 21:48:20 +08:00
|
|
|
|
|
|
|
if (skb->encapsulation)
|
|
|
|
skb->inner_transport_header = skb->transport_header;
|
|
|
|
|
2018-11-07 19:38:29 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-04-05 23:22:51 +08:00
|
|
|
int udp_gro_complete(struct sk_buff *skb, int nhoff,
|
|
|
|
udp_lookup_t lookup)
|
2014-01-20 19:59:19 +08:00
|
|
|
{
|
|
|
|
__be16 newlen = htons(skb->len - nhoff);
|
|
|
|
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
|
2016-04-05 23:22:51 +08:00
|
|
|
struct sock *sk;
|
2020-11-10 10:57:58 +08:00
|
|
|
int err;
|
2014-01-20 19:59:19 +08:00
|
|
|
|
|
|
|
uh->len = newlen;
|
|
|
|
|
2018-12-14 18:52:00 +08:00
|
|
|
sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
|
|
|
|
udp4_lib_lookup_skb, skb, uh->source, uh->dest);
|
2020-01-25 18:26:45 +08:00
|
|
|
if (sk && udp_sk(sk)->gro_complete) {
|
2018-11-07 19:38:29 +08:00
|
|
|
skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
|
|
|
|
: SKB_GSO_UDP_TUNNEL;
|
|
|
|
|
udp: properly complete L4 GRO over UDP tunnel packet
After the previous patch, the stack can do L4 UDP aggregation
on top of a UDP tunnel.
In such scenario, udp{4,6}_gro_complete will be called twice. This function
will enter its is_flist branch immediately, even though that is only
correct on the second call, as GSO_FRAGLIST is only relevant for the
inner packet.
Instead, we need to try first UDP tunnel-based aggregation, if the GRO
packet requires that.
This patch changes udp{4,6}_gro_complete to skip the frag list processing
when while encap_mark == 1, identifying processing of the outer tunnel
header.
Additionally, clears the field in udp_gro_complete() so that we can enter
the frag list path on the next round, for the inner header.
v1 -> v2:
- hopefully clarified the commit message
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-03-30 18:28:51 +08:00
|
|
|
/* clear the encap mark, so that inner frag_list gro_complete
|
|
|
|
* can take place
|
|
|
|
*/
|
|
|
|
NAPI_GRO_CB(skb)->encap_mark = 0;
|
|
|
|
|
2018-11-07 19:38:29 +08:00
|
|
|
/* Set encapsulation before calling into inner gro_complete()
|
|
|
|
* functions to make them set up the inner offsets.
|
|
|
|
*/
|
|
|
|
skb->encapsulation = 1;
|
2016-04-05 23:22:51 +08:00
|
|
|
err = udp_sk(sk)->gro_complete(sk, skb,
|
|
|
|
nhoff + sizeof(struct udphdr));
|
2020-01-25 18:26:45 +08:00
|
|
|
} else {
|
|
|
|
err = udp_gro_complete_segment(skb);
|
2018-11-07 19:38:29 +08:00
|
|
|
}
|
2015-02-11 08:30:29 +08:00
|
|
|
|
|
|
|
if (skb->remcsum_offload)
|
|
|
|
skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;
|
|
|
|
|
2014-01-20 19:59:19 +08:00
|
|
|
return err;
|
|
|
|
}
|
2016-04-05 23:22:51 +08:00
|
|
|
EXPORT_SYMBOL(udp_gro_complete);
|
2014-01-20 19:59:19 +08:00
|
|
|
|
2018-12-14 18:51:59 +08:00
|
|
|
INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
|
2014-08-23 04:34:44 +08:00
|
|
|
{
|
net: gro: fix udp bad offset in socket lookup by adding {inner_}network_offset to napi_gro_cb
[ Upstream commit 5ef31ea5d053a8f493a772ebad3f3ce82c35d845 ]
Commits a602456 ("udp: Add GRO functions to UDP socket") and 57c67ff ("udp:
additional GRO support") introduce incorrect usage of {ip,ipv6}_hdr in the
complete phase of gro. The functions always return skb->network_header,
which in the case of encapsulated packets at the gro complete phase, is
always set to the innermost L3 of the packet. That means that calling
{ip,ipv6}_hdr for skbs which completed the GRO receive phase (both in
gro_list and *_gro_complete) when parsing an encapsulated packet's _outer_
L3/L4 may return an unexpected value.
This incorrect usage leads to a bug in GRO's UDP socket lookup.
udp{4,6}_lib_lookup_skb functions use ip_hdr/ipv6_hdr respectively. These
*_hdr functions return network_header which will point to the innermost L3,
resulting in the wrong offset being used in __udp{4,6}_lib_lookup with
encapsulated packets.
This patch adds network_offset and inner_network_offset to napi_gro_cb, and
makes sure both are set correctly.
To fix the issue, network_offsets union is used inside napi_gro_cb, in
which both the outer and the inner network offsets are saved.
Reproduction example:
Endpoint configuration example (fou + local address bind)
# ip fou add port 6666 ipproto 4
# ip link add name tun1 type ipip remote 2.2.2.1 local 2.2.2.2 encap fou encap-dport 5555 encap-sport 6666 mode ipip
# ip link set tun1 up
# ip a add 1.1.1.2/24 dev tun1
Netperf TCP_STREAM result on net-next before patch is applied:
net-next main, GRO enabled:
$ netperf -H 1.1.1.2 -t TCP_STREAM -l 5
Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec
131072 16384 16384 5.28 2.37
net-next main, GRO disabled:
$ netperf -H 1.1.1.2 -t TCP_STREAM -l 5
Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec
131072 16384 16384 5.01 2745.06
patch applied, GRO enabled:
$ netperf -H 1.1.1.2 -t TCP_STREAM -l 5
Recv Send Send
Socket Socket Message Elapsed
Size Size Size Time Throughput
bytes bytes bytes secs. 10^6bits/sec
131072 16384 16384 5.01 2877.38
Fixes: a6024562ffd7 ("udp: Add GRO functions to UDP socket")
Signed-off-by: Richard Gobert <richardbgobert@gmail.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>
2024-04-30 22:35:54 +08:00
|
|
|
const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
|
|
|
|
const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
|
2014-08-23 04:34:44 +08:00
|
|
|
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
|
|
|
|
|
udp: properly complete L4 GRO over UDP tunnel packet
After the previous patch, the stack can do L4 UDP aggregation
on top of a UDP tunnel.
In such scenario, udp{4,6}_gro_complete will be called twice. This function
will enter its is_flist branch immediately, even though that is only
correct on the second call, as GSO_FRAGLIST is only relevant for the
inner packet.
Instead, we need to try first UDP tunnel-based aggregation, if the GRO
packet requires that.
This patch changes udp{4,6}_gro_complete to skip the frag list processing
when while encap_mark == 1, identifying processing of the outer tunnel
header.
Additionally, clears the field in udp_gro_complete() so that we can enter
the frag list path on the next round, for the inner header.
v1 -> v2:
- hopefully clarified the commit message
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-03-30 18:28:51 +08:00
|
|
|
/* do fraglist only if there is no outer UDP encap (or we already processed it) */
|
|
|
|
if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
|
2020-01-25 18:26:45 +08:00
|
|
|
uh->len = htons(skb->len - nhoff);
|
|
|
|
|
|
|
|
skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
|
|
|
|
skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
|
|
|
|
|
2024-03-26 19:34:00 +08:00
|
|
|
__skb_incr_checksum_unnecessary(skb);
|
2020-01-25 18:26:45 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-11-07 19:38:29 +08:00
|
|
|
if (uh->check)
|
2014-08-23 04:34:44 +08:00
|
|
|
uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
|
|
|
|
iph->daddr, 0);
|
|
|
|
|
2016-04-05 23:22:51 +08:00
|
|
|
return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
|
2014-08-23 04:34:44 +08:00
|
|
|
}
|
|
|
|
|
2013-06-08 18:56:03 +08:00
|
|
|
static const struct net_offload udpv4_offload = {
|
|
|
|
.callbacks = {
|
net: accept UFO datagrams from tuntap and packet
Tuntap and similar devices can inject GSO packets. Accept type
VIRTIO_NET_HDR_GSO_UDP, even though not generating UFO natively.
Processes are expected to use feature negotiation such as TUNSETOFFLOAD
to detect supported offload types and refrain from injecting other
packets. This process breaks down with live migration: guest kernels
do not renegotiate flags, so destination hosts need to expose all
features that the source host does.
Partially revert the UFO removal from 182e0b6b5846~1..d9d30adf5677.
This patch introduces nearly(*) no new code to simplify verification.
It brings back verbatim tuntap UFO negotiation, VIRTIO_NET_HDR_GSO_UDP
insertion and software UFO segmentation.
It does not reinstate protocol stack support, hardware offload
(NETIF_F_UFO), SKB_GSO_UDP tunneling in SKB_GSO_SOFTWARE or reception
of VIRTIO_NET_HDR_GSO_UDP packets in tuntap.
To support SKB_GSO_UDP reappearing in the stack, also reinstate
logic in act_csum and openvswitch. Achieve equivalence with v4.13 HEAD
by squashing in commit 939912216fa8 ("net: skb_needs_check() removes
CHECKSUM_UNNECESSARY check for tx.") and reverting commit 8d63bee643f1
("net: avoid skb_warn_bad_offload false positives on UFO").
(*) To avoid having to bring back skb_shinfo(skb)->ip6_frag_id,
ipv6_proxy_select_ident is changed to return a __be32 and this is
assigned directly to the frag_hdr. Also, SKB_GSO_UDP is inserted
at the end of the enum to minimize code churn.
Tested
Booted a v4.13 guest kernel with QEMU. On a host kernel before this
patch `ethtool -k eth0` shows UFO disabled. After the patch, it is
enabled, same as on a v4.13 host kernel.
A UFO packet sent from the guest appears on the tap device:
host:
nc -l -p -u 8000 &
tcpdump -n -i tap0
guest:
dd if=/dev/zero of=payload.txt bs=1 count=2000
nc -u 192.16.1.1 8000 < payload.txt
Direct tap to tap transmission of VIRTIO_NET_HDR_GSO_UDP succeeds,
packets arriving fragmented:
./with_tap_pair.sh ./tap_send_ufo tap0 tap1
(from https://github.com/wdebruij/kerneltools/tree/master/tests)
Changes
v1 -> v2
- simplified set_offload change (review comment)
- documented test procedure
Link: http://lkml.kernel.org/r/<CAF=yD-LuUeDuL9YWPJD9ykOZ0QCjNeznPDr6whqZ9NGMNF12Mw@mail.gmail.com>
Fixes: fb652fdfe837 ("macvlan/macvtap: Remove NETIF_F_UFO advertisement.")
Reported-by: Michal Kubecek <mkubecek@suse.cz>
Signed-off-by: Willem de Bruijn <willemb@google.com>
Acked-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-11-21 23:22:25 +08:00
|
|
|
.gso_segment = udp4_ufo_fragment,
|
2014-08-23 04:34:44 +08:00
|
|
|
.gro_receive = udp4_gro_receive,
|
|
|
|
.gro_complete = udp4_gro_complete,
|
2013-06-08 18:56:03 +08:00
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
int __init udpv4_offload_init(void)
|
|
|
|
{
|
|
|
|
return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
|
|
|
|
}
|