2019-05-29 22:12:43 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2013-10-04 09:16:47 +08:00
|
|
|
/*
|
2014-05-07 09:41:20 +08:00
|
|
|
* Copyright (c) 2007-2014 Nicira, Inc.
|
2013-10-04 09:16:47 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include "flow.h"
|
|
|
|
#include "datapath.h"
|
2015-07-21 16:44:03 +08:00
|
|
|
#include "flow_netlink.h"
|
2013-10-04 09:16:47 +08:00
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/etherdevice.h>
|
|
|
|
#include <linux/if_ether.h>
|
|
|
|
#include <linux/if_vlan.h>
|
|
|
|
#include <net/llc_pdu.h>
|
|
|
|
#include <linux/kernel.h>
|
2014-12-10 23:33:11 +08:00
|
|
|
#include <linux/jhash.h>
|
2013-10-04 09:16:47 +08:00
|
|
|
#include <linux/jiffies.h>
|
|
|
|
#include <linux/llc.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/in.h>
|
|
|
|
#include <linux/rcupdate.h>
|
2016-09-16 06:11:53 +08:00
|
|
|
#include <linux/cpumask.h>
|
2013-10-04 09:16:47 +08:00
|
|
|
#include <linux/if_arp.h>
|
|
|
|
#include <linux/ip.h>
|
|
|
|
#include <linux/ipv6.h>
|
|
|
|
#include <linux/sctp.h>
|
|
|
|
#include <linux/tcp.h>
|
|
|
|
#include <linux/udp.h>
|
|
|
|
#include <linux/icmp.h>
|
|
|
|
#include <linux/icmpv6.h>
|
|
|
|
#include <linux/rculist.h>
|
2020-07-15 20:09:28 +08:00
|
|
|
#include <linux/sort.h>
|
2013-10-04 09:16:47 +08:00
|
|
|
#include <net/ip.h>
|
|
|
|
#include <net/ipv6.h>
|
|
|
|
#include <net/ndisc.h>
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
#define TBL_MIN_BUCKETS 1024
|
2019-11-01 22:23:46 +08:00
|
|
|
#define MASK_ARRAY_SIZE_MIN 16
|
2013-10-04 15:14:23 +08:00
|
|
|
#define REHASH_INTERVAL (10 * 60 * HZ)
|
|
|
|
|
2020-07-31 20:21:34 +08:00
|
|
|
#define MC_DEFAULT_HASH_ENTRIES 256
|
2019-11-01 22:23:45 +08:00
|
|
|
#define MC_HASH_SHIFT 8
|
|
|
|
#define MC_HASH_SEGS ((sizeof(uint32_t) * 8) / MC_HASH_SHIFT)
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
static struct kmem_cache *flow_cache;
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
struct kmem_cache *flow_stats_cache __read_mostly;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
|
|
|
static u16 range_n_bytes(const struct sw_flow_key_range *range)
|
|
|
|
{
|
|
|
|
return range->end - range->start;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src,
|
2015-09-22 11:21:20 +08:00
|
|
|
bool full, const struct sw_flow_mask *mask)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2015-09-22 11:21:20 +08:00
|
|
|
int start = full ? 0 : mask->range.start;
|
|
|
|
int len = full ? sizeof *dst : range_n_bytes(&mask->range);
|
|
|
|
const long *m = (const long *)((const u8 *)&mask->key + start);
|
|
|
|
const long *s = (const long *)((const u8 *)src + start);
|
|
|
|
long *d = (long *)((u8 *)dst + start);
|
2013-10-04 09:16:47 +08:00
|
|
|
int i;
|
|
|
|
|
2015-09-22 11:21:20 +08:00
|
|
|
/* If 'full' is true then all of 'dst' is fully initialized. Otherwise,
|
|
|
|
* if 'full' is false the memory outside of the 'mask->range' is left
|
|
|
|
* uninitialized. This can be used as an optimization when further
|
|
|
|
* operations on 'dst' only use contents within 'mask->range'.
|
2013-10-04 09:16:47 +08:00
|
|
|
*/
|
2015-09-22 11:21:20 +08:00
|
|
|
for (i = 0; i < len; i += sizeof(long))
|
2013-10-04 09:16:47 +08:00
|
|
|
*d++ = *s++ & *m++;
|
|
|
|
}
|
|
|
|
|
2014-03-28 03:35:23 +08:00
|
|
|
struct sw_flow *ovs_flow_alloc(void)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
|
|
|
struct sw_flow *flow;
|
2019-07-20 00:20:13 +08:00
|
|
|
struct sw_flow_stats *stats;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2016-09-16 06:11:53 +08:00
|
|
|
flow = kmem_cache_zalloc(flow_cache, GFP_KERNEL);
|
2013-10-04 09:16:47 +08:00
|
|
|
if (!flow)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
2016-09-16 06:11:53 +08:00
|
|
|
flow->stats_last_writer = -1;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
/* Initialize the default stat node. */
|
|
|
|
stats = kmem_cache_alloc_node(flow_stats_cache,
|
2015-10-02 18:18:22 +08:00
|
|
|
GFP_KERNEL | __GFP_ZERO,
|
|
|
|
node_online(0) ? 0 : NUMA_NO_NODE);
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
if (!stats)
|
2014-03-28 03:35:23 +08:00
|
|
|
goto err;
|
2013-10-30 08:22:21 +08:00
|
|
|
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
spin_lock_init(&stats->lock);
|
|
|
|
|
|
|
|
RCU_INIT_POINTER(flow->stats[0], stats);
|
|
|
|
|
openvswitch: Optimize operations for OvS flow_stats.
When calling the flow_free() to free the flow, we call many times
(cpu_possible_mask, eg. 128 as default) cpumask_next(). That will
take up our CPU usage if we call the flow_free() frequently.
When we put all packets to userspace via upcall, and OvS will send
them back via netlink to ovs_packet_cmd_execute(will call flow_free).
The test topo is shown as below. VM01 sends TCP packets to VM02,
and OvS forward packtets. When testing, we use perf to report the
system performance.
VM01 --- OvS-VM --- VM02
Without this patch, perf-top show as below: The flow_free() is
3.02% CPU usage.
4.23% [kernel] [k] _raw_spin_unlock_irqrestore
3.62% [kernel] [k] __do_softirq
3.16% [kernel] [k] __memcpy
3.02% [kernel] [k] flow_free
2.42% libc-2.17.so [.] __memcpy_ssse3_back
2.18% [kernel] [k] copy_user_generic_unrolled
2.17% [kernel] [k] find_next_bit
When applied this patch, perf-top show as below: Not shown on
the list anymore.
4.11% [kernel] [k] _raw_spin_unlock_irqrestore
3.79% [kernel] [k] __do_softirq
3.46% [kernel] [k] __memcpy
2.73% libc-2.17.so [.] __memcpy_ssse3_back
2.25% [kernel] [k] copy_user_generic_unrolled
1.89% libc-2.17.so [.] _int_malloc
1.53% ovs-vswitchd [.] xlate_actions
With this patch, the TCP throughput(we dont use Megaflow Cache
+ Microflow Cache) between VMs is 1.18Gbs/sec up to 1.30Gbs/sec
(maybe ~10% performance imporve).
This patch adds cpumask struct, the cpu_used_mask stores the cpu_id
that the flow used. And we only check the flow_stats on the cpu we
used, and it is unncessary to check all possible cpu when getting,
cleaning, and updating the flow_stats. Adding the cpu_used_mask to
sw_flow struct does’t increase the cacheline number.
Signed-off-by: Tonghao Zhang <xiangxia.m.yue@gmail.com>
Acked-by: Pravin B Shelar <pshelar@ovn.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-18 14:28:06 +08:00
|
|
|
cpumask_set_cpu(0, &flow->cpu_used_mask);
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
return flow;
|
2013-10-30 08:22:21 +08:00
|
|
|
err:
|
2014-01-08 18:13:14 +08:00
|
|
|
kmem_cache_free(flow_cache, flow);
|
2013-10-30 08:22:21 +08:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2014-11-06 22:58:52 +08:00
|
|
|
int ovs_flow_tbl_count(const struct flow_table *table)
|
2013-10-04 15:14:23 +08:00
|
|
|
{
|
|
|
|
return table->count;
|
|
|
|
}
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
static void flow_free(struct sw_flow *flow)
|
|
|
|
{
|
2016-09-16 06:11:53 +08:00
|
|
|
int cpu;
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
if (ovs_identifier_is_key(&flow->id))
|
|
|
|
kfree(flow->id.unmasked_key);
|
2015-07-21 16:44:03 +08:00
|
|
|
if (flow->sf_acts)
|
2020-09-01 20:26:12 +08:00
|
|
|
ovs_nla_free_flow_actions((struct sw_flow_actions __force *)
|
|
|
|
flow->sf_acts);
|
2016-09-16 06:11:53 +08:00
|
|
|
/* We open code this to make sure cpu 0 is always considered */
|
2020-09-01 20:26:12 +08:00
|
|
|
for (cpu = 0; cpu < nr_cpu_ids;
|
|
|
|
cpu = cpumask_next(cpu, &flow->cpu_used_mask)) {
|
2016-09-16 06:11:53 +08:00
|
|
|
if (flow->stats[cpu])
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
kmem_cache_free(flow_stats_cache,
|
2019-07-20 00:20:13 +08:00
|
|
|
(struct sw_flow_stats __force *)flow->stats[cpu]);
|
2020-09-01 20:26:12 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
kmem_cache_free(flow_cache, flow);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rcu_free_flow_callback(struct rcu_head *rcu)
|
|
|
|
{
|
|
|
|
struct sw_flow *flow = container_of(rcu, struct sw_flow, rcu);
|
|
|
|
|
|
|
|
flow_free(flow);
|
|
|
|
}
|
|
|
|
|
2014-01-22 01:31:04 +08:00
|
|
|
void ovs_flow_free(struct sw_flow *flow, bool deferred)
|
2013-10-04 15:17:42 +08:00
|
|
|
{
|
2014-01-22 01:31:04 +08:00
|
|
|
if (!flow)
|
2013-10-04 15:17:42 +08:00
|
|
|
return;
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
if (deferred)
|
|
|
|
call_rcu(&flow->rcu, rcu_free_flow_callback);
|
|
|
|
else
|
|
|
|
flow_free(flow);
|
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
static void __table_instance_destroy(struct table_instance *ti)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2019-03-12 14:31:02 +08:00
|
|
|
kvfree(ti->buckets);
|
2013-10-04 15:14:23 +08:00
|
|
|
kfree(ti);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
static struct table_instance *table_instance_alloc(int new_size)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2013-10-04 15:14:23 +08:00
|
|
|
struct table_instance *ti = kmalloc(sizeof(*ti), GFP_KERNEL);
|
2019-03-12 14:31:02 +08:00
|
|
|
int i;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
if (!ti)
|
2013-10-04 09:16:47 +08:00
|
|
|
return NULL;
|
|
|
|
|
2019-03-12 14:31:02 +08:00
|
|
|
ti->buckets = kvmalloc_array(new_size, sizeof(struct hlist_head),
|
|
|
|
GFP_KERNEL);
|
2013-10-04 15:14:23 +08:00
|
|
|
if (!ti->buckets) {
|
|
|
|
kfree(ti);
|
2013-10-04 09:16:47 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
2019-03-12 14:31:02 +08:00
|
|
|
|
|
|
|
for (i = 0; i < new_size; i++)
|
|
|
|
INIT_HLIST_HEAD(&ti->buckets[i]);
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
ti->n_buckets = new_size;
|
|
|
|
ti->node_ver = 0;
|
|
|
|
get_random_bytes(&ti->hash_seed, sizeof(u32));
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
return ti;
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2020-07-15 20:09:28 +08:00
|
|
|
static void __mask_array_destroy(struct mask_array *ma)
|
|
|
|
{
|
2020-10-18 02:24:51 +08:00
|
|
|
free_percpu(ma->masks_usage_stats);
|
2020-07-15 20:09:28 +08:00
|
|
|
kfree(ma);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mask_array_rcu_cb(struct rcu_head *rcu)
|
|
|
|
{
|
|
|
|
struct mask_array *ma = container_of(rcu, struct mask_array, rcu);
|
|
|
|
|
|
|
|
__mask_array_destroy(ma);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tbl_mask_array_reset_counters(struct mask_array *ma)
|
|
|
|
{
|
|
|
|
int i, cpu;
|
|
|
|
|
|
|
|
/* As the per CPU counters are not atomic we can not go ahead and
|
|
|
|
* reset them from another CPU. To be able to still have an approximate
|
|
|
|
* zero based counter we store the value at reset, and subtract it
|
|
|
|
* later when processing.
|
|
|
|
*/
|
2020-09-01 20:26:12 +08:00
|
|
|
for (i = 0; i < ma->max; i++) {
|
2020-07-15 20:09:28 +08:00
|
|
|
ma->masks_usage_zero_cntr[i] = 0;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
2020-10-18 02:24:51 +08:00
|
|
|
struct mask_array_stats *stats;
|
2020-07-15 20:09:28 +08:00
|
|
|
unsigned int start;
|
|
|
|
u64 counter;
|
|
|
|
|
2020-10-18 02:24:51 +08:00
|
|
|
stats = per_cpu_ptr(ma->masks_usage_stats, cpu);
|
2020-07-15 20:09:28 +08:00
|
|
|
do {
|
2020-10-18 02:24:51 +08:00
|
|
|
start = u64_stats_fetch_begin_irq(&stats->syncp);
|
|
|
|
counter = stats->usage_cntrs[i];
|
|
|
|
} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
|
2020-07-15 20:09:28 +08:00
|
|
|
|
|
|
|
ma->masks_usage_zero_cntr[i] += counter;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:46 +08:00
|
|
|
static struct mask_array *tbl_mask_array_alloc(int size)
|
|
|
|
{
|
|
|
|
struct mask_array *new;
|
|
|
|
|
|
|
|
size = max(MASK_ARRAY_SIZE_MIN, size);
|
|
|
|
new = kzalloc(sizeof(struct mask_array) +
|
2020-07-15 20:09:28 +08:00
|
|
|
sizeof(struct sw_flow_mask *) * size +
|
|
|
|
sizeof(u64) * size, GFP_KERNEL);
|
2019-11-01 22:23:46 +08:00
|
|
|
if (!new)
|
|
|
|
return NULL;
|
|
|
|
|
2020-07-15 20:09:28 +08:00
|
|
|
new->masks_usage_zero_cntr = (u64 *)((u8 *)new +
|
|
|
|
sizeof(struct mask_array) +
|
|
|
|
sizeof(struct sw_flow_mask *) *
|
|
|
|
size);
|
|
|
|
|
2020-10-18 02:24:51 +08:00
|
|
|
new->masks_usage_stats = __alloc_percpu(sizeof(struct mask_array_stats) +
|
|
|
|
sizeof(u64) * size,
|
|
|
|
__alignof__(u64));
|
|
|
|
if (!new->masks_usage_stats) {
|
2020-07-15 20:09:28 +08:00
|
|
|
kfree(new);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:46 +08:00
|
|
|
new->count = 0;
|
|
|
|
new->max = size;
|
|
|
|
|
|
|
|
return new;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tbl_mask_array_realloc(struct flow_table *tbl, int size)
|
|
|
|
{
|
|
|
|
struct mask_array *old;
|
|
|
|
struct mask_array *new;
|
|
|
|
|
|
|
|
new = tbl_mask_array_alloc(size);
|
|
|
|
if (!new)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
old = ovsl_dereference(tbl->mask_array);
|
|
|
|
if (old) {
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < old->max; i++) {
|
|
|
|
if (ovsl_dereference(old->masks[i]))
|
|
|
|
new->masks[new->count++] = old->masks[i];
|
|
|
|
}
|
2020-07-15 20:09:28 +08:00
|
|
|
call_rcu(&old->rcu, mask_array_rcu_cb);
|
2019-11-01 22:23:46 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
rcu_assign_pointer(tbl->mask_array, new);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:52 +08:00
|
|
|
static int tbl_mask_array_add_mask(struct flow_table *tbl,
|
|
|
|
struct sw_flow_mask *new)
|
|
|
|
{
|
|
|
|
struct mask_array *ma = ovsl_dereference(tbl->mask_array);
|
|
|
|
int err, ma_count = READ_ONCE(ma->count);
|
|
|
|
|
|
|
|
if (ma_count >= ma->max) {
|
|
|
|
err = tbl_mask_array_realloc(tbl, ma->max +
|
2020-09-01 20:26:12 +08:00
|
|
|
MASK_ARRAY_SIZE_MIN);
|
2019-11-01 22:23:52 +08:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
ma = ovsl_dereference(tbl->mask_array);
|
2020-07-15 20:09:28 +08:00
|
|
|
} else {
|
|
|
|
/* On every add or delete we need to reset the counters so
|
|
|
|
* every new mask gets a fair chance of being prioritized.
|
|
|
|
*/
|
|
|
|
tbl_mask_array_reset_counters(ma);
|
2019-11-01 22:23:52 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
BUG_ON(ovsl_dereference(ma->masks[ma_count]));
|
|
|
|
|
|
|
|
rcu_assign_pointer(ma->masks[ma_count], new);
|
2020-09-01 20:26:12 +08:00
|
|
|
WRITE_ONCE(ma->count, ma_count + 1);
|
2019-11-01 22:23:52 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tbl_mask_array_del_mask(struct flow_table *tbl,
|
|
|
|
struct sw_flow_mask *mask)
|
|
|
|
{
|
|
|
|
struct mask_array *ma = ovsl_dereference(tbl->mask_array);
|
|
|
|
int i, ma_count = READ_ONCE(ma->count);
|
|
|
|
|
|
|
|
/* Remove the deleted mask pointers from the array */
|
|
|
|
for (i = 0; i < ma_count; i++) {
|
|
|
|
if (mask == ovsl_dereference(ma->masks[i]))
|
|
|
|
goto found;
|
|
|
|
}
|
|
|
|
|
|
|
|
BUG();
|
|
|
|
return;
|
|
|
|
|
|
|
|
found:
|
2020-09-01 20:26:12 +08:00
|
|
|
WRITE_ONCE(ma->count, ma_count - 1);
|
2019-11-01 22:23:52 +08:00
|
|
|
|
2020-09-01 20:26:12 +08:00
|
|
|
rcu_assign_pointer(ma->masks[i], ma->masks[ma_count - 1]);
|
|
|
|
RCU_INIT_POINTER(ma->masks[ma_count - 1], NULL);
|
2019-11-01 22:23:52 +08:00
|
|
|
|
|
|
|
kfree_rcu(mask, rcu);
|
|
|
|
|
|
|
|
/* Shrink the mask array if necessary. */
|
|
|
|
if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) &&
|
|
|
|
ma_count <= (ma->max / 3))
|
|
|
|
tbl_mask_array_realloc(tbl, ma->max / 2);
|
2020-07-15 20:09:28 +08:00
|
|
|
else
|
|
|
|
tbl_mask_array_reset_counters(ma);
|
|
|
|
|
2019-11-01 22:23:52 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove 'mask' from the mask list, if it is not needed any more. */
|
|
|
|
static void flow_mask_remove(struct flow_table *tbl, struct sw_flow_mask *mask)
|
|
|
|
{
|
|
|
|
if (mask) {
|
|
|
|
/* ovs-lock is required to protect mask-refcount and
|
|
|
|
* mask list.
|
|
|
|
*/
|
|
|
|
ASSERT_OVSL();
|
|
|
|
BUG_ON(!mask->ref_count);
|
|
|
|
mask->ref_count--;
|
|
|
|
|
|
|
|
if (!mask->ref_count)
|
|
|
|
tbl_mask_array_del_mask(tbl, mask);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-31 20:21:34 +08:00
|
|
|
static void __mask_cache_destroy(struct mask_cache *mc)
|
|
|
|
{
|
|
|
|
free_percpu(mc->mask_cache);
|
|
|
|
kfree(mc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mask_cache_rcu_cb(struct rcu_head *rcu)
|
|
|
|
{
|
|
|
|
struct mask_cache *mc = container_of(rcu, struct mask_cache, rcu);
|
|
|
|
|
|
|
|
__mask_cache_destroy(mc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct mask_cache *tbl_mask_cache_alloc(u32 size)
|
|
|
|
{
|
|
|
|
struct mask_cache_entry __percpu *cache = NULL;
|
|
|
|
struct mask_cache *new;
|
|
|
|
|
|
|
|
/* Only allow size to be 0, or a power of 2, and does not exceed
|
|
|
|
* percpu allocation size.
|
|
|
|
*/
|
|
|
|
if ((!is_power_of_2(size) && size != 0) ||
|
|
|
|
(size * sizeof(struct mask_cache_entry)) > PCPU_MIN_UNIT_SIZE)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
new = kzalloc(sizeof(*new), GFP_KERNEL);
|
|
|
|
if (!new)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
new->cache_size = size;
|
|
|
|
if (new->cache_size > 0) {
|
|
|
|
cache = __alloc_percpu(array_size(sizeof(struct mask_cache_entry),
|
|
|
|
new->cache_size),
|
|
|
|
__alignof__(struct mask_cache_entry));
|
|
|
|
if (!cache) {
|
|
|
|
kfree(new);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
new->mask_cache = cache;
|
|
|
|
return new;
|
|
|
|
}
|
|
|
|
int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size)
|
|
|
|
{
|
2020-11-03 16:25:49 +08:00
|
|
|
struct mask_cache *mc = rcu_dereference_ovsl(table->mask_cache);
|
2020-07-31 20:21:34 +08:00
|
|
|
struct mask_cache *new;
|
|
|
|
|
|
|
|
if (size == mc->cache_size)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if ((!is_power_of_2(size) && size != 0) ||
|
|
|
|
(size * sizeof(struct mask_cache_entry)) > PCPU_MIN_UNIT_SIZE)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
new = tbl_mask_cache_alloc(size);
|
|
|
|
if (!new)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
rcu_assign_pointer(table->mask_cache, new);
|
|
|
|
call_rcu(&mc->rcu, mask_cache_rcu_cb);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
int ovs_flow_tbl_init(struct flow_table *table)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2015-01-22 08:42:52 +08:00
|
|
|
struct table_instance *ti, *ufid_ti;
|
2020-07-31 20:21:34 +08:00
|
|
|
struct mask_cache *mc;
|
2019-11-01 22:23:46 +08:00
|
|
|
struct mask_array *ma;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2020-07-31 20:21:34 +08:00
|
|
|
mc = tbl_mask_cache_alloc(MC_DEFAULT_HASH_ENTRIES);
|
|
|
|
if (!mc)
|
2019-11-01 22:23:45 +08:00
|
|
|
return -ENOMEM;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2019-11-01 22:23:46 +08:00
|
|
|
ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);
|
|
|
|
if (!ma)
|
|
|
|
goto free_mask_cache;
|
|
|
|
|
2019-11-01 22:23:45 +08:00
|
|
|
ti = table_instance_alloc(TBL_MIN_BUCKETS);
|
2013-10-04 15:14:23 +08:00
|
|
|
if (!ti)
|
2019-11-01 22:23:46 +08:00
|
|
|
goto free_mask_array;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
ufid_ti = table_instance_alloc(TBL_MIN_BUCKETS);
|
|
|
|
if (!ufid_ti)
|
|
|
|
goto free_ti;
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
rcu_assign_pointer(table->ti, ti);
|
2015-01-22 08:42:52 +08:00
|
|
|
rcu_assign_pointer(table->ufid_ti, ufid_ti);
|
2019-11-01 22:23:46 +08:00
|
|
|
rcu_assign_pointer(table->mask_array, ma);
|
2020-07-31 20:21:34 +08:00
|
|
|
rcu_assign_pointer(table->mask_cache, mc);
|
2013-10-04 15:14:23 +08:00
|
|
|
table->last_rehash = jiffies;
|
|
|
|
table->count = 0;
|
2015-01-22 08:42:52 +08:00
|
|
|
table->ufid_count = 0;
|
2013-10-04 15:14:23 +08:00
|
|
|
return 0;
|
2015-01-22 08:42:52 +08:00
|
|
|
|
|
|
|
free_ti:
|
|
|
|
__table_instance_destroy(ti);
|
2019-11-01 22:23:46 +08:00
|
|
|
free_mask_array:
|
2020-07-15 20:09:28 +08:00
|
|
|
__mask_array_destroy(ma);
|
2019-11-01 22:23:45 +08:00
|
|
|
free_mask_cache:
|
2020-07-31 20:21:34 +08:00
|
|
|
__mask_cache_destroy(mc);
|
2015-01-22 08:42:52 +08:00
|
|
|
return -ENOMEM;
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_tbl_destroy_rcu_cb(struct rcu_head *rcu)
|
|
|
|
{
|
2020-09-01 20:26:12 +08:00
|
|
|
struct table_instance *ti;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2020-09-01 20:26:12 +08:00
|
|
|
ti = container_of(rcu, struct table_instance, rcu);
|
2013-10-04 15:14:23 +08:00
|
|
|
__table_instance_destroy(ti);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:52 +08:00
|
|
|
static void table_instance_flow_free(struct flow_table *table,
|
2020-09-01 20:26:12 +08:00
|
|
|
struct table_instance *ti,
|
|
|
|
struct table_instance *ufid_ti,
|
2020-09-01 20:26:13 +08:00
|
|
|
struct sw_flow *flow)
|
2019-11-01 22:23:52 +08:00
|
|
|
{
|
|
|
|
hlist_del_rcu(&flow->flow_table.node[ti->node_ver]);
|
2020-09-01 20:26:13 +08:00
|
|
|
table->count--;
|
2019-11-01 22:23:52 +08:00
|
|
|
|
|
|
|
if (ovs_identifier_is_ufid(&flow->id)) {
|
|
|
|
hlist_del_rcu(&flow->ufid_table.node[ufid_ti->node_ver]);
|
2020-09-01 20:26:13 +08:00
|
|
|
table->ufid_count--;
|
2019-11-01 22:23:52 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
flow_mask_remove(table, flow->mask);
|
|
|
|
}
|
|
|
|
|
2020-08-12 17:56:39 +08:00
|
|
|
/* Must be called with OVS mutex held. */
|
|
|
|
void table_instance_flow_flush(struct flow_table *table,
|
|
|
|
struct table_instance *ti,
|
|
|
|
struct table_instance *ufid_ti)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2014-01-22 01:31:04 +08:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < ti->n_buckets; i++) {
|
2019-03-12 14:31:02 +08:00
|
|
|
struct hlist_head *head = &ti->buckets[i];
|
2014-01-22 01:31:04 +08:00
|
|
|
struct hlist_node *n;
|
2020-09-01 20:26:12 +08:00
|
|
|
struct sw_flow *flow;
|
2014-01-22 01:31:04 +08:00
|
|
|
|
2019-11-01 22:23:52 +08:00
|
|
|
hlist_for_each_entry_safe(flow, n, head,
|
|
|
|
flow_table.node[ti->node_ver]) {
|
|
|
|
|
|
|
|
table_instance_flow_free(table, ti, ufid_ti,
|
2020-09-01 20:26:13 +08:00
|
|
|
flow);
|
2020-08-12 17:56:39 +08:00
|
|
|
ovs_flow_free(flow, true);
|
2014-01-22 01:31:04 +08:00
|
|
|
}
|
|
|
|
}
|
2020-09-01 20:26:13 +08:00
|
|
|
|
|
|
|
if (WARN_ON(table->count != 0 ||
|
|
|
|
table->ufid_count != 0)) {
|
|
|
|
table->count = 0;
|
|
|
|
table->ufid_count = 0;
|
|
|
|
}
|
2020-08-12 17:56:39 +08:00
|
|
|
}
|
2014-01-22 01:31:04 +08:00
|
|
|
|
2020-08-12 17:56:39 +08:00
|
|
|
static void table_instance_destroy(struct table_instance *ti,
|
|
|
|
struct table_instance *ufid_ti)
|
|
|
|
{
|
|
|
|
call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);
|
|
|
|
call_rcu(&ufid_ti->rcu, flow_tbl_destroy_rcu_cb);
|
2013-10-04 15:14:23 +08:00
|
|
|
}
|
|
|
|
|
2014-05-07 09:41:20 +08:00
|
|
|
/* No need for locking this function is called from RCU callback or
|
|
|
|
* error path.
|
|
|
|
*/
|
|
|
|
void ovs_flow_tbl_destroy(struct flow_table *table)
|
2013-10-04 15:14:23 +08:00
|
|
|
{
|
2014-05-07 09:41:20 +08:00
|
|
|
struct table_instance *ti = rcu_dereference_raw(table->ti);
|
2015-01-22 08:42:52 +08:00
|
|
|
struct table_instance *ufid_ti = rcu_dereference_raw(table->ufid_ti);
|
2020-08-05 15:19:11 +08:00
|
|
|
struct mask_cache *mc = rcu_dereference_raw(table->mask_cache);
|
|
|
|
struct mask_array *ma = rcu_dereference_raw(table->mask_array);
|
2013-10-04 15:14:23 +08:00
|
|
|
|
2020-07-31 20:21:34 +08:00
|
|
|
call_rcu(&mc->rcu, mask_cache_rcu_cb);
|
|
|
|
call_rcu(&ma->rcu, mask_array_rcu_cb);
|
2020-08-12 17:56:39 +08:00
|
|
|
table_instance_destroy(ti, ufid_ti);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti,
|
2013-10-04 09:16:47 +08:00
|
|
|
u32 *bucket, u32 *last)
|
|
|
|
{
|
|
|
|
struct sw_flow *flow;
|
|
|
|
struct hlist_head *head;
|
|
|
|
int ver;
|
|
|
|
int i;
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
ver = ti->node_ver;
|
|
|
|
while (*bucket < ti->n_buckets) {
|
2013-10-04 09:16:47 +08:00
|
|
|
i = 0;
|
2019-03-12 14:31:02 +08:00
|
|
|
head = &ti->buckets[*bucket];
|
2015-01-22 08:42:52 +08:00
|
|
|
hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) {
|
2013-10-04 09:16:47 +08:00
|
|
|
if (i < *last) {
|
|
|
|
i++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
*last = i + 1;
|
|
|
|
return flow;
|
|
|
|
}
|
|
|
|
(*bucket)++;
|
|
|
|
*last = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
static struct hlist_head *find_bucket(struct table_instance *ti, u32 hash)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2013-10-04 15:14:23 +08:00
|
|
|
hash = jhash_1word(hash, ti->hash_seed);
|
2019-03-12 14:31:02 +08:00
|
|
|
return &ti->buckets[hash & (ti->n_buckets - 1)];
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
static void table_instance_insert(struct table_instance *ti,
|
|
|
|
struct sw_flow *flow)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
|
|
|
struct hlist_head *head;
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
head = find_bucket(ti, flow->flow_table.hash);
|
|
|
|
hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void ufid_table_instance_insert(struct table_instance *ti,
|
|
|
|
struct sw_flow *flow)
|
|
|
|
{
|
|
|
|
struct hlist_head *head;
|
|
|
|
|
|
|
|
head = find_bucket(ti, flow->ufid_table.hash);
|
|
|
|
hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
static void flow_table_copy_flows(struct table_instance *old,
|
2015-01-22 08:42:52 +08:00
|
|
|
struct table_instance *new, bool ufid)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
|
|
|
int old_ver;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
old_ver = old->node_ver;
|
|
|
|
new->node_ver = !old_ver;
|
|
|
|
|
|
|
|
/* Insert in new table. */
|
|
|
|
for (i = 0; i < old->n_buckets; i++) {
|
|
|
|
struct sw_flow *flow;
|
2019-03-12 14:31:02 +08:00
|
|
|
struct hlist_head *head = &old->buckets[i];
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
if (ufid)
|
2020-03-26 04:27:24 +08:00
|
|
|
hlist_for_each_entry_rcu(flow, head,
|
|
|
|
ufid_table.node[old_ver],
|
|
|
|
lockdep_ovsl_is_held())
|
2015-01-22 08:42:52 +08:00
|
|
|
ufid_table_instance_insert(new, flow);
|
|
|
|
else
|
2020-03-26 04:27:24 +08:00
|
|
|
hlist_for_each_entry_rcu(flow, head,
|
|
|
|
flow_table.node[old_ver],
|
|
|
|
lockdep_ovsl_is_held())
|
2015-01-22 08:42:52 +08:00
|
|
|
table_instance_insert(new, flow);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
static struct table_instance *table_instance_rehash(struct table_instance *ti,
|
2015-01-22 08:42:52 +08:00
|
|
|
int n_buckets, bool ufid)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2013-10-04 15:14:23 +08:00
|
|
|
struct table_instance *new_ti;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
new_ti = table_instance_alloc(n_buckets);
|
|
|
|
if (!new_ti)
|
2013-10-04 15:17:42 +08:00
|
|
|
return NULL;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
flow_table_copy_flows(ti, new_ti, ufid);
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
return new_ti;
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
int ovs_flow_tbl_flush(struct flow_table *flow_table)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2015-01-22 08:42:52 +08:00
|
|
|
struct table_instance *old_ti, *new_ti;
|
|
|
|
struct table_instance *old_ufid_ti, *new_ufid_ti;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
new_ti = table_instance_alloc(TBL_MIN_BUCKETS);
|
|
|
|
if (!new_ti)
|
|
|
|
return -ENOMEM;
|
2015-01-22 08:42:52 +08:00
|
|
|
new_ufid_ti = table_instance_alloc(TBL_MIN_BUCKETS);
|
|
|
|
if (!new_ufid_ti)
|
|
|
|
goto err_free_ti;
|
|
|
|
|
|
|
|
old_ti = ovsl_dereference(flow_table->ti);
|
|
|
|
old_ufid_ti = ovsl_dereference(flow_table->ufid_ti);
|
2013-10-04 15:14:23 +08:00
|
|
|
|
|
|
|
rcu_assign_pointer(flow_table->ti, new_ti);
|
2015-01-22 08:42:52 +08:00
|
|
|
rcu_assign_pointer(flow_table->ufid_ti, new_ufid_ti);
|
2013-10-04 15:14:23 +08:00
|
|
|
flow_table->last_rehash = jiffies;
|
|
|
|
|
2020-08-12 17:56:39 +08:00
|
|
|
table_instance_flow_flush(flow_table, old_ti, old_ufid_ti);
|
|
|
|
table_instance_destroy(old_ti, old_ufid_ti);
|
2013-10-04 15:14:23 +08:00
|
|
|
return 0;
|
2015-01-22 08:42:52 +08:00
|
|
|
|
|
|
|
err_free_ti:
|
|
|
|
__table_instance_destroy(new_ti);
|
|
|
|
return -ENOMEM;
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2015-01-22 08:42:50 +08:00
|
|
|
static u32 flow_hash(const struct sw_flow_key *key,
|
|
|
|
const struct sw_flow_key_range *range)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2019-11-01 22:23:50 +08:00
|
|
|
const u32 *hash_key = (const u32 *)((const u8 *)key + range->start);
|
2013-10-04 09:16:47 +08:00
|
|
|
|
|
|
|
/* Make sure number of hash bytes are multiple of u32. */
|
2019-11-01 22:23:50 +08:00
|
|
|
int hash_u32s = range_n_bytes(range) >> 2;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2014-12-10 23:33:11 +08:00
|
|
|
return jhash2(hash_key, hash_u32s, 0);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static int flow_key_start(const struct sw_flow_key *key)
|
|
|
|
{
|
2015-10-05 19:09:46 +08:00
|
|
|
if (key->tun_proto)
|
2013-10-04 09:16:47 +08:00
|
|
|
return 0;
|
|
|
|
else
|
|
|
|
return rounddown(offsetof(struct sw_flow_key, phy),
|
2020-09-01 20:26:12 +08:00
|
|
|
sizeof(long));
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static bool cmp_key(const struct sw_flow_key *key1,
|
|
|
|
const struct sw_flow_key *key2,
|
|
|
|
int key_start, int key_end)
|
|
|
|
{
|
2014-01-24 02:56:49 +08:00
|
|
|
const long *cp1 = (const long *)((const u8 *)key1 + key_start);
|
|
|
|
const long *cp2 = (const long *)((const u8 *)key2 + key_start);
|
2013-10-04 09:16:47 +08:00
|
|
|
long diffs = 0;
|
|
|
|
int i;
|
|
|
|
|
2020-09-01 20:26:12 +08:00
|
|
|
for (i = key_start; i < key_end; i += sizeof(long))
|
2013-10-04 09:16:47 +08:00
|
|
|
diffs |= *cp1++ ^ *cp2++;
|
|
|
|
|
|
|
|
return diffs == 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool flow_cmp_masked_key(const struct sw_flow *flow,
|
|
|
|
const struct sw_flow_key *key,
|
2015-01-22 08:42:50 +08:00
|
|
|
const struct sw_flow_key_range *range)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2015-01-22 08:42:50 +08:00
|
|
|
return cmp_key(&flow->key, key, range->start, range->end);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
static bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow,
|
|
|
|
const struct sw_flow_match *match)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
|
|
|
struct sw_flow_key *key = match->key;
|
|
|
|
int key_start = flow_key_start(key);
|
|
|
|
int key_end = match->range.end;
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
BUG_ON(ovs_identifier_is_ufid(&flow->id));
|
|
|
|
return cmp_key(flow->id.unmasked_key, key, key_start, key_end);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
static struct sw_flow *masked_flow_lookup(struct table_instance *ti,
|
2013-10-04 09:16:47 +08:00
|
|
|
const struct sw_flow_key *unmasked,
|
2019-11-01 22:23:45 +08:00
|
|
|
const struct sw_flow_mask *mask,
|
|
|
|
u32 *n_mask_hit)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
|
|
|
struct sw_flow *flow;
|
|
|
|
struct hlist_head *head;
|
|
|
|
u32 hash;
|
|
|
|
struct sw_flow_key masked_key;
|
|
|
|
|
2015-09-22 11:21:20 +08:00
|
|
|
ovs_flow_mask_key(&masked_key, unmasked, false, mask);
|
2015-01-22 08:42:50 +08:00
|
|
|
hash = flow_hash(&masked_key, &mask->range);
|
2013-10-04 15:14:23 +08:00
|
|
|
head = find_bucket(ti, hash);
|
2019-11-01 22:23:45 +08:00
|
|
|
(*n_mask_hit)++;
|
|
|
|
|
2020-02-19 03:58:20 +08:00
|
|
|
hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver],
|
2020-09-01 20:26:12 +08:00
|
|
|
lockdep_ovsl_is_held()) {
|
2015-01-22 08:42:52 +08:00
|
|
|
if (flow->mask == mask && flow->flow_table.hash == hash &&
|
2015-01-22 08:42:50 +08:00
|
|
|
flow_cmp_masked_key(flow, &masked_key, &mask->range))
|
2013-10-04 09:16:47 +08:00
|
|
|
return flow;
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:48 +08:00
|
|
|
/* Flow lookup does full lookup on flow table. It starts with
|
|
|
|
* mask from index passed in *index.
|
2020-10-18 02:24:51 +08:00
|
|
|
* This function MUST be called with BH disabled due to the use
|
|
|
|
* of CPU specific variables.
|
2019-11-01 22:23:48 +08:00
|
|
|
*/
|
2019-11-01 22:23:45 +08:00
|
|
|
static struct sw_flow *flow_lookup(struct flow_table *tbl,
|
|
|
|
struct table_instance *ti,
|
2019-11-01 22:23:46 +08:00
|
|
|
struct mask_array *ma,
|
2019-11-01 22:23:45 +08:00
|
|
|
const struct sw_flow_key *key,
|
2019-11-01 22:23:46 +08:00
|
|
|
u32 *n_mask_hit,
|
2020-07-31 20:20:56 +08:00
|
|
|
u32 *n_cache_hit,
|
2019-11-01 22:23:46 +08:00
|
|
|
u32 *index)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2020-10-18 02:24:51 +08:00
|
|
|
struct mask_array_stats *stats = this_cpu_ptr(ma->masks_usage_stats);
|
2013-10-04 15:14:23 +08:00
|
|
|
struct sw_flow *flow;
|
2019-11-01 22:23:49 +08:00
|
|
|
struct sw_flow_mask *mask;
|
2019-11-01 22:23:46 +08:00
|
|
|
int i;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2019-11-01 22:23:51 +08:00
|
|
|
if (likely(*index < ma->max)) {
|
2019-11-01 22:23:48 +08:00
|
|
|
mask = rcu_dereference_ovsl(ma->masks[*index]);
|
2019-11-01 22:23:46 +08:00
|
|
|
if (mask) {
|
|
|
|
flow = masked_flow_lookup(ti, key, mask, n_mask_hit);
|
2020-07-15 20:09:28 +08:00
|
|
|
if (flow) {
|
2020-10-18 02:24:51 +08:00
|
|
|
u64_stats_update_begin(&stats->syncp);
|
|
|
|
stats->usage_cntrs[*index]++;
|
|
|
|
u64_stats_update_end(&stats->syncp);
|
2020-07-31 20:20:56 +08:00
|
|
|
(*n_cache_hit)++;
|
2019-11-01 22:23:46 +08:00
|
|
|
return flow;
|
2020-07-15 20:09:28 +08:00
|
|
|
}
|
2019-11-01 22:23:48 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < ma->max; i++) {
|
|
|
|
|
|
|
|
if (i == *index)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
mask = rcu_dereference_ovsl(ma->masks[i]);
|
2019-11-01 22:23:51 +08:00
|
|
|
if (unlikely(!mask))
|
2019-11-01 22:23:49 +08:00
|
|
|
break;
|
2019-11-01 22:23:48 +08:00
|
|
|
|
|
|
|
flow = masked_flow_lookup(ti, key, mask, n_mask_hit);
|
|
|
|
if (flow) { /* Found */
|
|
|
|
*index = i;
|
2020-10-18 02:24:51 +08:00
|
|
|
u64_stats_update_begin(&stats->syncp);
|
|
|
|
stats->usage_cntrs[*index]++;
|
|
|
|
u64_stats_update_end(&stats->syncp);
|
2019-11-01 22:23:48 +08:00
|
|
|
return flow;
|
2019-11-01 22:23:46 +08:00
|
|
|
}
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
2019-11-01 22:23:46 +08:00
|
|
|
|
2013-10-04 15:14:23 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2019-11-01 22:23:45 +08:00
|
|
|
/*
|
|
|
|
* mask_cache maps flow to probable mask. This cache is not tightly
|
|
|
|
* coupled cache, It means updates to mask list can result in inconsistent
|
|
|
|
* cache entry in mask cache.
|
|
|
|
* This is per cpu cache and is divided in MC_HASH_SEGS segments.
|
|
|
|
* In case of a hash collision the entry is hashed in next segment.
|
|
|
|
* */
|
|
|
|
struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl,
|
|
|
|
const struct sw_flow_key *key,
|
|
|
|
u32 skb_hash,
|
2020-07-31 20:20:56 +08:00
|
|
|
u32 *n_mask_hit,
|
|
|
|
u32 *n_cache_hit)
|
2019-11-01 22:23:45 +08:00
|
|
|
{
|
2020-07-31 20:21:34 +08:00
|
|
|
struct mask_cache *mc = rcu_dereference(tbl->mask_cache);
|
2019-11-01 22:23:48 +08:00
|
|
|
struct mask_array *ma = rcu_dereference(tbl->mask_array);
|
|
|
|
struct table_instance *ti = rcu_dereference(tbl->ti);
|
|
|
|
struct mask_cache_entry *entries, *ce;
|
2019-11-01 22:23:45 +08:00
|
|
|
struct sw_flow *flow;
|
2019-11-01 22:23:48 +08:00
|
|
|
u32 hash;
|
2019-11-01 22:23:45 +08:00
|
|
|
int seg;
|
|
|
|
|
|
|
|
*n_mask_hit = 0;
|
2020-07-31 20:20:56 +08:00
|
|
|
*n_cache_hit = 0;
|
2020-07-31 20:21:34 +08:00
|
|
|
if (unlikely(!skb_hash || mc->cache_size == 0)) {
|
2019-11-01 22:23:48 +08:00
|
|
|
u32 mask_index = 0;
|
2020-07-31 20:20:56 +08:00
|
|
|
u32 cache = 0;
|
2019-11-01 22:23:46 +08:00
|
|
|
|
2020-07-31 20:20:56 +08:00
|
|
|
return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,
|
|
|
|
&mask_index);
|
2019-11-01 22:23:46 +08:00
|
|
|
}
|
2019-11-01 22:23:45 +08:00
|
|
|
|
2019-11-01 22:23:48 +08:00
|
|
|
/* Pre and post recirulation flows usually have the same skb_hash
|
|
|
|
* value. To avoid hash collisions, rehash the 'skb_hash' with
|
|
|
|
* 'recirc_id'. */
|
|
|
|
if (key->recirc_id)
|
|
|
|
skb_hash = jhash_1word(skb_hash, key->recirc_id);
|
|
|
|
|
|
|
|
ce = NULL;
|
|
|
|
hash = skb_hash;
|
2020-07-31 20:21:34 +08:00
|
|
|
entries = this_cpu_ptr(mc->mask_cache);
|
2019-11-01 22:23:45 +08:00
|
|
|
|
2019-11-01 22:23:48 +08:00
|
|
|
/* Find the cache entry 'ce' to operate on. */
|
2019-11-01 22:23:45 +08:00
|
|
|
for (seg = 0; seg < MC_HASH_SEGS; seg++) {
|
2020-07-31 20:21:34 +08:00
|
|
|
int index = hash & (mc->cache_size - 1);
|
2019-11-01 22:23:48 +08:00
|
|
|
struct mask_cache_entry *e;
|
|
|
|
|
|
|
|
e = &entries[index];
|
|
|
|
if (e->skb_hash == skb_hash) {
|
|
|
|
flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,
|
2020-07-31 20:20:56 +08:00
|
|
|
n_cache_hit, &e->mask_index);
|
2019-11-01 22:23:48 +08:00
|
|
|
if (!flow)
|
|
|
|
e->skb_hash = 0;
|
|
|
|
return flow;
|
2019-11-01 22:23:45 +08:00
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:48 +08:00
|
|
|
if (!ce || e->skb_hash < ce->skb_hash)
|
|
|
|
ce = e; /* A better replacement cache candidate. */
|
2019-11-01 22:23:45 +08:00
|
|
|
|
|
|
|
hash >>= MC_HASH_SHIFT;
|
|
|
|
}
|
|
|
|
|
2019-11-01 22:23:48 +08:00
|
|
|
/* Cache miss, do full lookup. */
|
2020-07-31 20:20:56 +08:00
|
|
|
flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,
|
|
|
|
&ce->mask_index);
|
2019-11-01 22:23:46 +08:00
|
|
|
if (flow)
|
2019-11-01 22:23:48 +08:00
|
|
|
ce->skb_hash = skb_hash;
|
2019-11-01 22:23:45 +08:00
|
|
|
|
2020-07-31 20:20:56 +08:00
|
|
|
*n_cache_hit = 0;
|
2019-11-01 22:23:45 +08:00
|
|
|
return flow;
|
|
|
|
}
|
|
|
|
|
2013-11-26 02:42:46 +08:00
|
|
|
struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *tbl,
|
|
|
|
const struct sw_flow_key *key)
|
|
|
|
{
|
2019-11-01 22:23:45 +08:00
|
|
|
struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);
|
2019-11-01 22:23:46 +08:00
|
|
|
struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);
|
2013-11-26 02:42:46 +08:00
|
|
|
u32 __always_unused n_mask_hit;
|
2020-07-31 20:20:56 +08:00
|
|
|
u32 __always_unused n_cache_hit;
|
2020-10-18 02:24:51 +08:00
|
|
|
struct sw_flow *flow;
|
2019-11-01 22:23:48 +08:00
|
|
|
u32 index = 0;
|
2013-11-26 02:42:46 +08:00
|
|
|
|
2020-10-18 02:24:51 +08:00
|
|
|
/* This function gets called trough the netlink interface and therefore
|
|
|
|
* is preemptible. However, flow_lookup() function needs to be called
|
|
|
|
* with BH disabled due to CPU specific variables.
|
|
|
|
*/
|
|
|
|
local_bh_disable();
|
|
|
|
flow = flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);
|
|
|
|
local_bh_enable();
|
|
|
|
return flow;
|
2013-11-26 02:42:46 +08:00
|
|
|
}
|
|
|
|
|
2014-07-01 11:30:29 +08:00
|
|
|
struct sw_flow *ovs_flow_tbl_lookup_exact(struct flow_table *tbl,
|
2014-11-06 22:58:52 +08:00
|
|
|
const struct sw_flow_match *match)
|
2014-07-01 11:30:29 +08:00
|
|
|
{
|
2019-11-01 22:23:46 +08:00
|
|
|
struct mask_array *ma = ovsl_dereference(tbl->mask_array);
|
|
|
|
int i;
|
2014-07-01 11:30:29 +08:00
|
|
|
|
|
|
|
/* Always called under ovs-mutex. */
|
2019-11-01 22:23:46 +08:00
|
|
|
for (i = 0; i < ma->max; i++) {
|
|
|
|
struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);
|
|
|
|
u32 __always_unused n_mask_hit;
|
|
|
|
struct sw_flow_mask *mask;
|
|
|
|
struct sw_flow *flow;
|
|
|
|
|
|
|
|
mask = ovsl_dereference(ma->masks[i]);
|
|
|
|
if (!mask)
|
|
|
|
continue;
|
|
|
|
|
2019-11-01 22:23:45 +08:00
|
|
|
flow = masked_flow_lookup(ti, match->key, mask, &n_mask_hit);
|
2015-01-22 08:42:52 +08:00
|
|
|
if (flow && ovs_identifier_is_key(&flow->id) &&
|
2019-11-01 22:23:46 +08:00
|
|
|
ovs_flow_cmp_unmasked_key(flow, match)) {
|
2015-01-22 08:42:52 +08:00
|
|
|
return flow;
|
2019-11-01 22:23:46 +08:00
|
|
|
}
|
2015-01-22 08:42:52 +08:00
|
|
|
}
|
2019-11-01 22:23:46 +08:00
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 ufid_hash(const struct sw_flow_id *sfid)
|
|
|
|
{
|
|
|
|
return jhash(sfid->ufid, sfid->ufid_len, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool ovs_flow_cmp_ufid(const struct sw_flow *flow,
|
|
|
|
const struct sw_flow_id *sfid)
|
|
|
|
{
|
|
|
|
if (flow->id.ufid_len != sfid->ufid_len)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return !memcmp(flow->id.ufid, sfid->ufid, sfid->ufid_len);
|
|
|
|
}
|
|
|
|
|
2020-09-01 20:26:12 +08:00
|
|
|
bool ovs_flow_cmp(const struct sw_flow *flow,
|
|
|
|
const struct sw_flow_match *match)
|
2015-01-22 08:42:52 +08:00
|
|
|
{
|
|
|
|
if (ovs_identifier_is_ufid(&flow->id))
|
|
|
|
return flow_cmp_masked_key(flow, match->key, &match->range);
|
|
|
|
|
|
|
|
return ovs_flow_cmp_unmasked_key(flow, match);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct sw_flow *ovs_flow_tbl_lookup_ufid(struct flow_table *tbl,
|
|
|
|
const struct sw_flow_id *ufid)
|
|
|
|
{
|
|
|
|
struct table_instance *ti = rcu_dereference_ovsl(tbl->ufid_ti);
|
|
|
|
struct sw_flow *flow;
|
|
|
|
struct hlist_head *head;
|
|
|
|
u32 hash;
|
|
|
|
|
|
|
|
hash = ufid_hash(ufid);
|
|
|
|
head = find_bucket(ti, hash);
|
2020-02-19 03:58:20 +08:00
|
|
|
hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver],
|
2020-09-01 20:26:12 +08:00
|
|
|
lockdep_ovsl_is_held()) {
|
2015-01-22 08:42:52 +08:00
|
|
|
if (flow->ufid_table.hash == hash &&
|
|
|
|
ovs_flow_cmp_ufid(flow, ufid))
|
2014-07-01 11:30:29 +08:00
|
|
|
return flow;
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2013-10-23 01:42:46 +08:00
|
|
|
int ovs_flow_tbl_num_masks(const struct flow_table *table)
|
|
|
|
{
|
2019-11-01 22:23:46 +08:00
|
|
|
struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);
|
2019-11-01 22:23:49 +08:00
|
|
|
return READ_ONCE(ma->count);
|
2013-10-23 01:42:46 +08:00
|
|
|
}
|
|
|
|
|
2020-07-31 20:21:34 +08:00
|
|
|
u32 ovs_flow_tbl_masks_cache_size(const struct flow_table *table)
|
|
|
|
{
|
2020-08-05 15:19:11 +08:00
|
|
|
struct mask_cache *mc = rcu_dereference_ovsl(table->mask_cache);
|
2020-07-31 20:21:34 +08:00
|
|
|
|
|
|
|
return READ_ONCE(mc->cache_size);
|
|
|
|
}
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
static struct table_instance *table_instance_expand(struct table_instance *ti,
|
|
|
|
bool ufid)
|
2013-10-04 15:14:23 +08:00
|
|
|
{
|
2015-01-22 08:42:52 +08:00
|
|
|
return table_instance_rehash(ti, ti->n_buckets * 2, ufid);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2014-05-06 04:24:53 +08:00
|
|
|
/* Must be called with OVS mutex held. */
|
2013-10-04 09:16:47 +08:00
|
|
|
void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)
|
|
|
|
{
|
2013-10-04 15:14:23 +08:00
|
|
|
struct table_instance *ti = ovsl_dereference(table->ti);
|
2015-01-22 08:42:52 +08:00
|
|
|
struct table_instance *ufid_ti = ovsl_dereference(table->ufid_ti);
|
2013-10-04 15:14:23 +08:00
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
BUG_ON(table->count == 0);
|
2020-09-01 20:26:13 +08:00
|
|
|
table_instance_flow_free(table, ti, ufid_ti, flow);
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:17:42 +08:00
|
|
|
static struct sw_flow_mask *mask_alloc(void)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
|
|
|
struct sw_flow_mask *mask;
|
|
|
|
|
|
|
|
mask = kmalloc(sizeof(*mask), GFP_KERNEL);
|
|
|
|
if (mask)
|
2014-01-22 01:31:04 +08:00
|
|
|
mask->ref_count = 1;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
|
|
|
return mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool mask_equal(const struct sw_flow_mask *a,
|
|
|
|
const struct sw_flow_mask *b)
|
|
|
|
{
|
2014-01-24 02:56:49 +08:00
|
|
|
const u8 *a_ = (const u8 *)&a->key + a->range.start;
|
|
|
|
const u8 *b_ = (const u8 *)&b->key + b->range.start;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
|
|
|
return (a->range.end == b->range.end)
|
|
|
|
&& (a->range.start == b->range.start)
|
|
|
|
&& (memcmp(a_, b_, range_n_bytes(&a->range)) == 0);
|
|
|
|
}
|
|
|
|
|
2013-10-04 15:17:42 +08:00
|
|
|
static struct sw_flow_mask *flow_mask_find(const struct flow_table *tbl,
|
2013-10-04 09:16:47 +08:00
|
|
|
const struct sw_flow_mask *mask)
|
|
|
|
{
|
2019-11-01 22:23:46 +08:00
|
|
|
struct mask_array *ma;
|
|
|
|
int i;
|
2013-10-04 09:16:47 +08:00
|
|
|
|
2019-11-01 22:23:46 +08:00
|
|
|
ma = ovsl_dereference(tbl->mask_array);
|
|
|
|
for (i = 0; i < ma->max; i++) {
|
|
|
|
struct sw_flow_mask *t;
|
|
|
|
t = ovsl_dereference(ma->masks[i]);
|
|
|
|
|
|
|
|
if (t && mask_equal(mask, t))
|
|
|
|
return t;
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2013-11-26 02:40:51 +08:00
|
|
|
/* Add 'mask' into the mask list, if it is not already there. */
|
2013-10-04 15:17:42 +08:00
|
|
|
static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow,
|
2014-11-06 22:58:52 +08:00
|
|
|
const struct sw_flow_mask *new)
|
2013-10-04 15:17:42 +08:00
|
|
|
{
|
|
|
|
struct sw_flow_mask *mask;
|
2019-11-01 22:23:46 +08:00
|
|
|
|
2013-10-04 15:17:42 +08:00
|
|
|
mask = flow_mask_find(tbl, new);
|
|
|
|
if (!mask) {
|
|
|
|
/* Allocate a new mask if none exsits. */
|
|
|
|
mask = mask_alloc();
|
|
|
|
if (!mask)
|
|
|
|
return -ENOMEM;
|
|
|
|
mask->key = new->key;
|
|
|
|
mask->range = new->range;
|
2019-11-01 22:23:46 +08:00
|
|
|
|
|
|
|
/* Add mask to mask-list. */
|
2019-11-01 22:23:49 +08:00
|
|
|
if (tbl_mask_array_add_mask(tbl, mask)) {
|
|
|
|
kfree(mask);
|
|
|
|
return -ENOMEM;
|
2019-11-01 22:23:46 +08:00
|
|
|
}
|
2014-01-22 01:31:04 +08:00
|
|
|
} else {
|
|
|
|
BUG_ON(!mask->ref_count);
|
|
|
|
mask->ref_count++;
|
2013-10-04 15:17:42 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
flow->mask = mask;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-05-06 04:24:53 +08:00
|
|
|
/* Must be called with OVS mutex held. */
|
2015-01-22 08:42:49 +08:00
|
|
|
static void flow_key_insert(struct flow_table *table, struct sw_flow *flow)
|
2013-10-04 09:16:47 +08:00
|
|
|
{
|
2013-10-04 15:17:42 +08:00
|
|
|
struct table_instance *new_ti = NULL;
|
|
|
|
struct table_instance *ti;
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
flow->flow_table.hash = flow_hash(&flow->key, &flow->mask->range);
|
2013-10-04 15:17:42 +08:00
|
|
|
ti = ovsl_dereference(table->ti);
|
|
|
|
table_instance_insert(ti, flow);
|
|
|
|
table->count++;
|
|
|
|
|
|
|
|
/* Expand table, if necessary, to make room. */
|
|
|
|
if (table->count > ti->n_buckets)
|
2015-01-22 08:42:52 +08:00
|
|
|
new_ti = table_instance_expand(ti, false);
|
2013-10-04 15:17:42 +08:00
|
|
|
else if (time_after(jiffies, table->last_rehash + REHASH_INTERVAL))
|
2015-01-22 08:42:52 +08:00
|
|
|
new_ti = table_instance_rehash(ti, ti->n_buckets, false);
|
2013-10-04 15:17:42 +08:00
|
|
|
|
|
|
|
if (new_ti) {
|
|
|
|
rcu_assign_pointer(table->ti, new_ti);
|
2015-01-22 08:42:52 +08:00
|
|
|
call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);
|
2013-10-04 15:17:42 +08:00
|
|
|
table->last_rehash = jiffies;
|
|
|
|
}
|
2015-01-22 08:42:49 +08:00
|
|
|
}
|
|
|
|
|
2015-01-22 08:42:52 +08:00
|
|
|
/* Must be called with OVS mutex held. */
|
|
|
|
static void flow_ufid_insert(struct flow_table *table, struct sw_flow *flow)
|
|
|
|
{
|
|
|
|
struct table_instance *ti;
|
|
|
|
|
|
|
|
flow->ufid_table.hash = ufid_hash(&flow->id);
|
|
|
|
ti = ovsl_dereference(table->ufid_ti);
|
|
|
|
ufid_table_instance_insert(ti, flow);
|
|
|
|
table->ufid_count++;
|
|
|
|
|
|
|
|
/* Expand table, if necessary, to make room. */
|
|
|
|
if (table->ufid_count > ti->n_buckets) {
|
|
|
|
struct table_instance *new_ti;
|
|
|
|
|
|
|
|
new_ti = table_instance_expand(ti, true);
|
|
|
|
if (new_ti) {
|
|
|
|
rcu_assign_pointer(table->ufid_ti, new_ti);
|
|
|
|
call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-01-22 08:42:49 +08:00
|
|
|
/* Must be called with OVS mutex held. */
|
|
|
|
int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,
|
|
|
|
const struct sw_flow_mask *mask)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = flow_mask_insert(table, flow, mask);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
flow_key_insert(table, flow);
|
2015-01-22 08:42:52 +08:00
|
|
|
if (ovs_identifier_is_ufid(&flow->id))
|
|
|
|
flow_ufid_insert(table, flow);
|
2015-01-22 08:42:49 +08:00
|
|
|
|
2013-10-04 15:17:42 +08:00
|
|
|
return 0;
|
2013-10-04 09:16:47 +08:00
|
|
|
}
|
|
|
|
|
2020-07-15 20:09:28 +08:00
|
|
|
static int compare_mask_and_count(const void *a, const void *b)
|
|
|
|
{
|
|
|
|
const struct mask_count *mc_a = a;
|
|
|
|
const struct mask_count *mc_b = b;
|
|
|
|
|
|
|
|
return (s64)mc_b->counter - (s64)mc_a->counter;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Must be called with OVS mutex held. */
|
|
|
|
void ovs_flow_masks_rebalance(struct flow_table *table)
|
|
|
|
{
|
|
|
|
struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);
|
|
|
|
struct mask_count *masks_and_count;
|
|
|
|
struct mask_array *new;
|
|
|
|
int masks_entries = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/* Build array of all current entries with use counters. */
|
|
|
|
masks_and_count = kmalloc_array(ma->max, sizeof(*masks_and_count),
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!masks_and_count)
|
|
|
|
return;
|
|
|
|
|
2020-09-01 20:26:12 +08:00
|
|
|
for (i = 0; i < ma->max; i++) {
|
2020-07-15 20:09:28 +08:00
|
|
|
struct sw_flow_mask *mask;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
mask = rcu_dereference_ovsl(ma->masks[i]);
|
|
|
|
if (unlikely(!mask))
|
|
|
|
break;
|
|
|
|
|
|
|
|
masks_and_count[i].index = i;
|
|
|
|
masks_and_count[i].counter = 0;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
2020-10-18 02:24:51 +08:00
|
|
|
struct mask_array_stats *stats;
|
|
|
|
unsigned int start;
|
2020-07-15 20:09:28 +08:00
|
|
|
u64 counter;
|
|
|
|
|
2020-10-18 02:24:51 +08:00
|
|
|
stats = per_cpu_ptr(ma->masks_usage_stats, cpu);
|
2020-07-15 20:09:28 +08:00
|
|
|
do {
|
2020-10-18 02:24:51 +08:00
|
|
|
start = u64_stats_fetch_begin_irq(&stats->syncp);
|
|
|
|
counter = stats->usage_cntrs[i];
|
|
|
|
} while (u64_stats_fetch_retry_irq(&stats->syncp,
|
|
|
|
start));
|
2020-07-15 20:09:28 +08:00
|
|
|
|
|
|
|
masks_and_count[i].counter += counter;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Subtract the zero count value. */
|
|
|
|
masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i];
|
|
|
|
|
|
|
|
/* Rather than calling tbl_mask_array_reset_counters()
|
|
|
|
* below when no change is needed, do it inline here.
|
|
|
|
*/
|
|
|
|
ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i == 0)
|
|
|
|
goto free_mask_entries;
|
|
|
|
|
|
|
|
/* Sort the entries */
|
|
|
|
masks_entries = i;
|
|
|
|
sort(masks_and_count, masks_entries, sizeof(*masks_and_count),
|
|
|
|
compare_mask_and_count, NULL);
|
|
|
|
|
|
|
|
/* If the order is the same, nothing to do... */
|
|
|
|
for (i = 0; i < masks_entries; i++) {
|
|
|
|
if (i != masks_and_count[i].index)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (i == masks_entries)
|
|
|
|
goto free_mask_entries;
|
|
|
|
|
|
|
|
/* Rebuilt the new list in order of usage. */
|
|
|
|
new = tbl_mask_array_alloc(ma->max);
|
|
|
|
if (!new)
|
|
|
|
goto free_mask_entries;
|
|
|
|
|
|
|
|
for (i = 0; i < masks_entries; i++) {
|
|
|
|
int index = masks_and_count[i].index;
|
|
|
|
|
2020-07-31 20:21:34 +08:00
|
|
|
if (ovsl_dereference(ma->masks[index]))
|
|
|
|
new->masks[new->count++] = ma->masks[index];
|
2020-07-15 20:09:28 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
rcu_assign_pointer(table->mask_array, new);
|
|
|
|
call_rcu(&ma->rcu, mask_array_rcu_cb);
|
|
|
|
|
|
|
|
free_mask_entries:
|
|
|
|
kfree(masks_and_count);
|
|
|
|
}
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
/* Initializes the flow module.
|
|
|
|
* Returns zero if successful or a negative error code. */
|
|
|
|
int ovs_flow_init(void)
|
|
|
|
{
|
|
|
|
BUILD_BUG_ON(__alignof__(struct sw_flow_key) % __alignof__(long));
|
|
|
|
BUILD_BUG_ON(sizeof(struct sw_flow_key) % sizeof(long));
|
|
|
|
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow)
|
2016-09-16 06:11:53 +08:00
|
|
|
+ (nr_cpu_ids
|
2019-07-20 00:20:13 +08:00
|
|
|
* sizeof(struct sw_flow_stats *)),
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
0, 0, NULL);
|
2013-10-04 09:16:47 +08:00
|
|
|
if (flow_cache == NULL)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
flow_stats_cache
|
2019-07-20 00:20:13 +08:00
|
|
|
= kmem_cache_create("sw_flow_stats", sizeof(struct sw_flow_stats),
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
0, SLAB_HWCACHE_ALIGN, NULL);
|
|
|
|
if (flow_stats_cache == NULL) {
|
|
|
|
kmem_cache_destroy(flow_cache);
|
|
|
|
flow_cache = NULL;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2013-10-04 09:16:47 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Uninitializes the flow module. */
|
|
|
|
void ovs_flow_exit(void)
|
|
|
|
{
|
openvswitch: Per NUMA node flow stats.
Keep kernel flow stats for each NUMA node rather than each (logical)
CPU. This avoids using the per-CPU allocator and removes most of the
kernel-side OVS locking overhead otherwise on the top of perf reports
and allows OVS to scale better with higher number of threads.
With 9 handlers and 4 revalidators netperf TCP_CRR test flow setup
rate doubles on a server with two hyper-threaded physical CPUs (16
logical cores each) compared to the current OVS master. Tested with
non-trivial flow table with a TCP port match rule forcing all new
connections with unique port numbers to OVS userspace. The IP
addresses are still wildcarded, so the kernel flows are not considered
as exact match 5-tuple flows. This type of flows can be expected to
appear in large numbers as the result of more effective wildcarding
made possible by improvements in OVS userspace flow classifier.
Perf results for this test (master):
Events: 305K cycles
+ 8.43% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 5.64% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 4.75% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 3.32% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 2.61% ovs-vswitchd [kernel.kallsyms] [k] pcpu_alloc_area
+ 2.19% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.03% swapper [kernel.kallsyms] [k] intel_idle
+ 1.84% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 1.64% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.58% ovs-vswitchd libc-2.15.so [.] 0x7f4e6
+ 1.07% ovs-vswitchd [kernel.kallsyms] [k] memset
+ 1.03% netperf [kernel.kallsyms] [k] __ticket_spin_lock
+ 0.92% swapper [kernel.kallsyms] [k] __ticket_spin_lock
...
And after this patch:
Events: 356K cycles
+ 6.85% ovs-vswitchd ovs-vswitchd [.] find_match_wc
+ 4.63% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_lock
+ 3.06% ovs-vswitchd [kernel.kallsyms] [k] __ticket_spin_lock
+ 2.81% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask_range
+ 2.51% ovs-vswitchd libpthread-2.15.so [.] pthread_mutex_unlock
+ 2.27% ovs-vswitchd ovs-vswitchd [.] classifier_lookup
+ 1.84% ovs-vswitchd libc-2.15.so [.] 0x15d30f
+ 1.74% ovs-vswitchd [kernel.kallsyms] [k] mutex_spin_on_owner
+ 1.47% swapper [kernel.kallsyms] [k] intel_idle
+ 1.34% ovs-vswitchd ovs-vswitchd [.] flow_hash_in_minimask
+ 1.33% ovs-vswitchd ovs-vswitchd [.] rule_actions_unref
+ 1.16% ovs-vswitchd ovs-vswitchd [.] hindex_node_with_hash
+ 1.16% ovs-vswitchd ovs-vswitchd [.] do_xlate_actions
+ 1.09% ovs-vswitchd ovs-vswitchd [.] ofproto_rule_ref
+ 1.01% netperf [kernel.kallsyms] [k] __ticket_spin_lock
...
There is a small increase in kernel spinlock overhead due to the same
spinlock being shared between multiple cores of the same physical CPU,
but that is barely visible in the netperf TCP_CRR test performance
(maybe ~1% performance drop, hard to tell exactly due to variance in
the test results), when testing for kernel module throughput (with no
userspace activity, handful of kernel flows).
On flow setup, a single stats instance is allocated (for the NUMA node
0). As CPUs from multiple NUMA nodes start updating stats, new
NUMA-node specific stats instances are allocated. This allocation on
the packet processing code path is made to never block or look for
emergency memory pools, minimizing the allocation latency. If the
allocation fails, the existing preallocated stats instance is used.
Also, if only CPUs from one NUMA-node are updating the preallocated
stats instance, no additional stats instances are allocated. This
eliminates the need to pre-allocate stats instances that will not be
used, also relieving the stats reader from the burden of reading stats
that are never used.
Signed-off-by: Jarno Rajahalme <jrajahalme@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: Jesse Gross <jesse@nicira.com>
2014-03-28 03:42:54 +08:00
|
|
|
kmem_cache_destroy(flow_stats_cache);
|
2013-10-04 09:16:47 +08:00
|
|
|
kmem_cache_destroy(flow_cache);
|
|
|
|
}
|