ipv6: move inet6_sk(sk)->rx_dst_cookie to sk->sk_rx_dst_cookie
Increase cache locality by moving rx_dst_coookie next to sk->sk_rx_dst This removes one or two cache line misses in IPv6 early demux (TCP/UDP) Signed-off-by: Eric Dumazet <edumazet@google.com> Acked-by: Soheil Hassas Yeganeh <soheil@google.com> Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
parent
0c0a5ef809
commit
ef57c1610d
|
@ -282,7 +282,6 @@ struct ipv6_pinfo {
|
||||||
__be32 rcv_flowinfo;
|
__be32 rcv_flowinfo;
|
||||||
|
|
||||||
__u32 dst_cookie;
|
__u32 dst_cookie;
|
||||||
__u32 rx_dst_cookie;
|
|
||||||
|
|
||||||
struct ipv6_mc_socklist __rcu *ipv6_mc_list;
|
struct ipv6_mc_socklist __rcu *ipv6_mc_list;
|
||||||
struct ipv6_ac_socklist *ipv6_ac_list;
|
struct ipv6_ac_socklist *ipv6_ac_list;
|
||||||
|
|
|
@ -260,6 +260,7 @@ struct bpf_local_storage;
|
||||||
* @sk_wq: sock wait queue and async head
|
* @sk_wq: sock wait queue and async head
|
||||||
* @sk_rx_dst: receive input route used by early demux
|
* @sk_rx_dst: receive input route used by early demux
|
||||||
* @sk_rx_dst_ifindex: ifindex for @sk_rx_dst
|
* @sk_rx_dst_ifindex: ifindex for @sk_rx_dst
|
||||||
|
* @sk_rx_dst_cookie: cookie for @sk_rx_dst
|
||||||
* @sk_dst_cache: destination cache
|
* @sk_dst_cache: destination cache
|
||||||
* @sk_dst_pending_confirm: need to confirm neighbour
|
* @sk_dst_pending_confirm: need to confirm neighbour
|
||||||
* @sk_policy: flow policy
|
* @sk_policy: flow policy
|
||||||
|
@ -432,6 +433,7 @@ struct sock {
|
||||||
#endif
|
#endif
|
||||||
struct dst_entry *sk_rx_dst;
|
struct dst_entry *sk_rx_dst;
|
||||||
int sk_rx_dst_ifindex;
|
int sk_rx_dst_ifindex;
|
||||||
|
u32 sk_rx_dst_cookie;
|
||||||
|
|
||||||
struct dst_entry __rcu *sk_dst_cache;
|
struct dst_entry __rcu *sk_dst_cache;
|
||||||
atomic_t sk_omem_alloc;
|
atomic_t sk_omem_alloc;
|
||||||
|
|
|
@ -109,7 +109,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
|
||||||
|
|
||||||
sk->sk_rx_dst = dst;
|
sk->sk_rx_dst = dst;
|
||||||
sk->sk_rx_dst_ifindex = skb->skb_iif;
|
sk->sk_rx_dst_ifindex = skb->skb_iif;
|
||||||
tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
|
sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1511,7 +1511,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
|
||||||
if (dst) {
|
if (dst) {
|
||||||
if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
|
if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
|
||||||
INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
|
INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
|
||||||
dst, np->rx_dst_cookie) == NULL) {
|
dst, sk->sk_rx_dst_cookie) == NULL) {
|
||||||
dst_release(dst);
|
dst_release(dst);
|
||||||
sk->sk_rx_dst = NULL;
|
sk->sk_rx_dst = NULL;
|
||||||
}
|
}
|
||||||
|
@ -1872,7 +1872,7 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)
|
||||||
struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
|
struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
|
||||||
|
|
||||||
if (dst)
|
if (dst)
|
||||||
dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
|
dst = dst_check(dst, sk->sk_rx_dst_cookie);
|
||||||
if (dst &&
|
if (dst &&
|
||||||
sk->sk_rx_dst_ifindex == skb->skb_iif)
|
sk->sk_rx_dst_ifindex == skb->skb_iif)
|
||||||
skb_dst_set_noref(skb, dst);
|
skb_dst_set_noref(skb, dst);
|
||||||
|
|
|
@ -884,7 +884,7 @@ static void udp6_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
|
||||||
if (udp_sk_rx_dst_set(sk, dst)) {
|
if (udp_sk_rx_dst_set(sk, dst)) {
|
||||||
const struct rt6_info *rt = (const struct rt6_info *)dst;
|
const struct rt6_info *rt = (const struct rt6_info *)dst;
|
||||||
|
|
||||||
inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
|
sk->sk_rx_dst_cookie = rt6_get_cookie(rt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1073,7 +1073,7 @@ INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb)
|
||||||
dst = READ_ONCE(sk->sk_rx_dst);
|
dst = READ_ONCE(sk->sk_rx_dst);
|
||||||
|
|
||||||
if (dst)
|
if (dst)
|
||||||
dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie);
|
dst = dst_check(dst, sk->sk_rx_dst_cookie);
|
||||||
if (dst) {
|
if (dst) {
|
||||||
/* set noref for now.
|
/* set noref for now.
|
||||||
* any place which wants to hold dst has to call
|
* any place which wants to hold dst has to call
|
||||||
|
|
Loading…
Reference in New Issue