2006-06-30 16:55:32 +08:00
|
|
|
#ifndef _LINUX_VMSTAT_H
|
|
|
|
|
#define _LINUX_VMSTAT_H
|
|
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
|
#include <linux/percpu.h>
|
2007-02-10 17:43:03 +08:00
|
|
|
#include <linux/mm.h>
|
2006-06-30 16:55:33 +08:00
|
|
|
#include <linux/mmzone.h>
|
2011-05-27 07:25:24 +08:00
|
|
|
#include <linux/vm_event_item.h>
|
2011-07-27 07:09:06 +08:00
|
|
|
#include <linux/atomic.h>
|
2006-06-30 16:55:32 +08:00
|
|
|
|
2008-07-24 12:27:03 +08:00
|
|
|
extern int sysctl_stat_interval;
|
|
|
|
|
|
2007-02-10 17:44:41 +08:00
|
|
|
#ifdef CONFIG_VM_EVENT_COUNTERS
|
|
|
|
|
/*
|
|
|
|
|
* Light weight per cpu counter implementation.
|
|
|
|
|
*
|
|
|
|
|
* Counters should only be incremented and no critical kernel component
|
|
|
|
|
* should rely on the counter values.
|
|
|
|
|
*
|
|
|
|
|
* Counters are handled completely inline. On many platforms the code
|
|
|
|
|
* generated will simply be the increment of a global address.
|
|
|
|
|
*/
|
|
|
|
|
|
2006-06-30 16:55:45 +08:00
|
|
|
struct vm_event_state {
|
|
|
|
|
unsigned long event[NR_VM_EVENT_ITEMS];
|
2006-06-30 16:55:32 +08:00
|
|
|
};
|
|
|
|
|
|
2006-06-30 16:55:45 +08:00
|
|
|
DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
|
|
|
|
|
|
2014-04-08 06:39:43 +08:00
|
|
|
/*
|
|
|
|
|
* vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
|
|
|
|
|
* local_irq_disable overhead.
|
|
|
|
|
*/
|
2006-06-30 16:55:45 +08:00
|
|
|
static inline void __count_vm_event(enum vm_event_item item)
|
|
|
|
|
{
|
2014-04-08 06:39:43 +08:00
|
|
|
raw_cpu_inc(vm_event_states.event[item]);
|
2006-06-30 16:55:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void count_vm_event(enum vm_event_item item)
|
|
|
|
|
{
|
2009-10-29 21:34:15 +08:00
|
|
|
this_cpu_inc(vm_event_states.event[item]);
|
2006-06-30 16:55:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void __count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
|
{
|
2014-04-08 06:39:43 +08:00
|
|
|
raw_cpu_add(vm_event_states.event[item], delta);
|
2006-06-30 16:55:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
|
{
|
2009-10-29 21:34:15 +08:00
|
|
|
this_cpu_add(vm_event_states.event[item], delta);
|
2006-06-30 16:55:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extern void all_vm_events(unsigned long *);
|
2013-04-30 06:08:14 +08:00
|
|
|
|
2006-06-30 16:55:45 +08:00
|
|
|
extern void vm_events_fold_cpu(int cpu);
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
|
|
/* Disable counters */
|
2007-02-10 17:44:41 +08:00
|
|
|
static inline void count_vm_event(enum vm_event_item item)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void __count_vm_event(enum vm_event_item item)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void __count_vm_events(enum vm_event_item item, long delta)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void all_vm_events(unsigned long *ret)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void vm_events_fold_cpu(int cpu)
|
|
|
|
|
{
|
|
|
|
|
}
|
2006-06-30 16:55:45 +08:00
|
|
|
|
|
|
|
|
#endif /* CONFIG_VM_EVENT_COUNTERS */
|
|
|
|
|
|
2012-11-02 22:52:48 +08:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
|
#define count_vm_numa_event(x) count_vm_event(x)
|
|
|
|
|
#define count_vm_numa_events(x, y) count_vm_events(x, y)
|
|
|
|
|
#else
|
|
|
|
|
#define count_vm_numa_event(x) do {} while (0)
|
2013-02-23 08:34:29 +08:00
|
|
|
#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
|
2012-11-02 22:52:48 +08:00
|
|
|
#endif /* CONFIG_NUMA_BALANCING */
|
|
|
|
|
|
2014-01-22 06:33:16 +08:00
|
|
|
#ifdef CONFIG_DEBUG_TLBFLUSH
|
|
|
|
|
#define count_vm_tlb_event(x) count_vm_event(x)
|
|
|
|
|
#define count_vm_tlb_events(x, y) count_vm_events(x, y)
|
|
|
|
|
#else
|
|
|
|
|
#define count_vm_tlb_event(x) do {} while (0)
|
|
|
|
|
#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
2014-06-05 07:06:46 +08:00
|
|
|
#ifdef CONFIG_DEBUG_VM_VMACACHE
|
|
|
|
|
#define count_vm_vmacache_event(x) count_vm_event(x)
|
|
|
|
|
#else
|
|
|
|
|
#define count_vm_vmacache_event(x) do {} while (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
2006-06-30 16:55:45 +08:00
|
|
|
#define __count_zone_vm_events(item, zone, delta) \
|
2007-02-10 17:43:10 +08:00
|
|
|
__count_vm_events(item##_NORMAL - ZONE_NORMAL + \
|
|
|
|
|
zone_idx(zone), delta)
|
2006-06-30 16:55:32 +08:00
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
/*
|
|
|
|
|
* Zone based page accounting with per cpu differentials.
|
|
|
|
|
*/
|
|
|
|
|
extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
|
|
|
|
|
|
|
|
|
|
static inline void zone_page_state_add(long x, struct zone *zone,
|
|
|
|
|
enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
atomic_long_add(x, &zone->vm_stat[item]);
|
|
|
|
|
atomic_long_add(x, &vm_stat[item]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long global_page_state(enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
long x = atomic_long_read(&vm_stat[item]);
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
if (x < 0)
|
|
|
|
|
x = 0;
|
|
|
|
|
#endif
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long zone_page_state(struct zone *zone,
|
|
|
|
|
enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
long x = atomic_long_read(&zone->vm_stat[item]);
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
if (x < 0)
|
|
|
|
|
x = 0;
|
|
|
|
|
#endif
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
2010-09-10 07:38:17 +08:00
|
|
|
/*
|
|
|
|
|
* More accurate version that also considers the currently pending
|
|
|
|
|
* deltas. For that we need to loop over all cpus to find the current
|
|
|
|
|
* deltas. There is no synchronization so the result cannot be
|
|
|
|
|
* exactly accurate either.
|
|
|
|
|
*/
|
|
|
|
|
static inline unsigned long zone_page_state_snapshot(struct zone *zone,
|
|
|
|
|
enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
long x = atomic_long_read(&zone->vm_stat[item]);
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
int cpu;
|
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
|
x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
|
|
|
|
|
|
|
|
|
|
if (x < 0)
|
|
|
|
|
x = 0;
|
|
|
|
|
#endif
|
|
|
|
|
return x;
|
|
|
|
|
}
|
|
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
|
/*
|
|
|
|
|
* Determine the per node value of a stat item. This function
|
|
|
|
|
* is called frequently in a NUMA machine, so try to be as
|
|
|
|
|
* frugal as possible.
|
|
|
|
|
*/
|
|
|
|
|
static inline unsigned long node_page_state(int node,
|
|
|
|
|
enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
struct zone *zones = NODE_DATA(node)->node_zones;
|
|
|
|
|
|
|
|
|
|
return
|
2007-02-10 17:43:10 +08:00
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
|
|
|
zone_page_state(&zones[ZONE_DMA], item) +
|
|
|
|
|
#endif
|
2006-09-26 14:31:13 +08:00
|
|
|
#ifdef CONFIG_ZONE_DMA32
|
2006-06-30 16:55:33 +08:00
|
|
|
zone_page_state(&zones[ZONE_DMA32], item) +
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
|
zone_page_state(&zones[ZONE_HIGHMEM], item) +
|
|
|
|
|
#endif
|
2007-07-17 19:03:12 +08:00
|
|
|
zone_page_state(&zones[ZONE_NORMAL], item) +
|
|
|
|
|
zone_page_state(&zones[ZONE_MOVABLE], item);
|
2006-06-30 16:55:33 +08:00
|
|
|
}
|
2006-06-30 16:55:44 +08:00
|
|
|
|
2011-03-23 07:33:12 +08:00
|
|
|
extern void zone_statistics(struct zone *, struct zone *, gfp_t gfp);
|
2006-06-30 16:55:44 +08:00
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
#else
|
2006-06-30 16:55:44 +08:00
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
#define node_page_state(node, item) global_page_state(item)
|
2011-03-23 07:33:12 +08:00
|
|
|
#define zone_statistics(_zl, _z, gfp) do { } while (0)
|
2006-06-30 16:55:44 +08:00
|
|
|
|
|
|
|
|
#endif /* CONFIG_NUMA */
|
2006-06-30 16:55:33 +08:00
|
|
|
|
|
|
|
|
#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
|
|
|
|
|
#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
|
|
|
|
|
void __inc_zone_page_state(struct page *, enum zone_stat_item);
|
|
|
|
|
void __dec_zone_page_state(struct page *, enum zone_stat_item);
|
2006-06-30 16:55:32 +08:00
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
|
|
|
|
|
void inc_zone_page_state(struct page *, enum zone_stat_item);
|
|
|
|
|
void dec_zone_page_state(struct page *, enum zone_stat_item);
|
|
|
|
|
|
|
|
|
|
extern void inc_zone_state(struct zone *, enum zone_stat_item);
|
2007-02-10 17:43:01 +08:00
|
|
|
extern void __inc_zone_state(struct zone *, enum zone_stat_item);
|
|
|
|
|
extern void dec_zone_state(struct zone *, enum zone_stat_item);
|
|
|
|
|
extern void __dec_zone_state(struct zone *, enum zone_stat_item);
|
2006-06-30 16:55:33 +08:00
|
|
|
|
2013-09-12 05:21:30 +08:00
|
|
|
void cpu_vm_stats_fold(int cpu);
|
2011-05-25 08:11:33 +08:00
|
|
|
void refresh_zone_stat_thresholds(void);
|
2011-01-14 07:45:43 +08:00
|
|
|
|
2012-10-09 07:33:39 +08:00
|
|
|
void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
|
|
|
|
|
|
2011-01-14 07:45:43 +08:00
|
|
|
int calculate_pressure_threshold(struct zone *zone);
|
|
|
|
|
int calculate_normal_threshold(struct zone *zone);
|
|
|
|
|
void set_pgdat_percpu_threshold(pg_data_t *pgdat,
|
|
|
|
|
int (*calculate_pressure)(struct zone *));
|
2006-06-30 16:55:33 +08:00
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We do not maintain differentials in a single processor configuration.
|
|
|
|
|
* The functions directly modify the zone and global counters.
|
|
|
|
|
*/
|
|
|
|
|
static inline void __mod_zone_page_state(struct zone *zone,
|
|
|
|
|
enum zone_stat_item item, int delta)
|
|
|
|
|
{
|
|
|
|
|
zone_page_state_add(delta, zone, item);
|
|
|
|
|
}
|
|
|
|
|
|
2006-07-10 19:44:30 +08:00
|
|
|
static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
atomic_long_inc(&zone->vm_stat[item]);
|
|
|
|
|
atomic_long_inc(&vm_stat[item]);
|
|
|
|
|
}
|
|
|
|
|
|
2007-02-10 17:43:01 +08:00
|
|
|
static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
atomic_long_dec(&zone->vm_stat[item]);
|
|
|
|
|
atomic_long_dec(&vm_stat[item]);
|
|
|
|
|
}
|
|
|
|
|
|
mm: vmstat: fix UP zone state accounting
Summary:
The VM maintains cached filesystem pages on two types of lists. One
list holds the pages recently faulted into the cache, the other list
holds pages that have been referenced repeatedly on that first list.
The idea is to prefer reclaiming young pages over those that have shown
to benefit from caching in the past. We call the recently used list
"inactive list" and the frequently used list "active list".
Currently, the VM aims for a 1:1 ratio between the lists, which is the
"perfect" trade-off between the ability to *protect* frequently used
pages and the ability to *detect* frequently used pages. This means
that working set changes bigger than half of cache memory go undetected
and thrash indefinitely, whereas working sets bigger than half of cache
memory are unprotected against used-once streams that don't even need
caching.
This happens on file servers and media streaming servers, where the
popular files and file sections change over time. Even though the
individual files might be smaller than half of memory, concurrent access
to many of them may still result in their inter-reference distance being
greater than half of memory. It's also been reported as a problem on
database workloads that switch back and forth between tables that are
bigger than half of memory. In these cases the VM never recognizes the
new working set and will for the remainder of the workload thrash disk
data which could easily live in memory.
Historically, every reclaim scan of the inactive list also took a
smaller number of pages from the tail of the active list and moved them
to the head of the inactive list. This model gave established working
sets more gracetime in the face of temporary use-once streams, but
ultimately was not significantly better than a FIFO policy and still
thrashed cache based on eviction speed, rather than actual demand for
cache.
This series solves the problem by maintaining a history of pages evicted
from the inactive list, enabling the VM to detect frequently used pages
regardless of inactive list size and facilitate working set transitions.
Tests:
The reported database workload is easily demonstrated on a 8G machine
with two filesets a 6G. This fio workload operates on one set first,
then switches to the other. The VM should obviously always cache the
set that the workload is currently using.
This test is based on a problem encountered by Citus Data customers:
http://citusdata.com/blog/72-linux-memory-manager-and-your-big-data
unpatched:
db1: READ: io=98304MB, aggrb=885559KB/s, minb=885559KB/s, maxb=885559KB/s, mint= 113672msec, maxt= 113672msec
db2: READ: io=98304MB, aggrb= 66169KB/s, minb= 66169KB/s, maxb= 66169KB/s, mint=1521302msec, maxt=1521302msec
sdb: ios=835750/4, merge=2/1, ticks=4659739/60016, in_queue=4719203, util=98.92%
real 27m15.541s
user 0m19.059s
sys 0m51.459s
patched:
db1: READ: io=98304MB, aggrb=877783KB/s, minb=877783KB/s, maxb=877783KB/s, mint=114679msec, maxt=114679msec
db2: READ: io=98304MB, aggrb=397449KB/s, minb=397449KB/s, maxb=397449KB/s, mint=253273msec, maxt=253273msec
sdb: ios=170587/4, merge=2/1, ticks=954910/61123, in_queue=1015923, util=90.40%
real 6m8.630s
user 0m14.714s
sys 0m31.233s
As can be seen, the unpatched kernel simply never adapts to the
workingset change and db2 is stuck indefinitely with secondary storage
speed. The patched kernel needs 2-3 iterations over db2 before it
replaces db1 and reaches full memory speed. Given the unbounded
negative affect of the existing VM behavior, these patches should be
considered correctness fixes rather than performance optimizations.
Another test resembles a fileserver or streaming server workload, where
data in excess of memory size is accessed at different frequencies.
There is very hot data accessed at a high frequency. Machines should be
fitted so that the hot set of such a workload can be fully cached or all
bets are off. Then there is a very big (compared to available memory)
set of data that is used-once or at a very low frequency; this is what
drives the inactive list and does not really benefit from caching.
Lastly, there is a big set of warm data in between that is accessed at
medium frequencies and benefits from caching the pages between the first
and last streamer of each burst.
unpatched:
hot: READ: io=128000MB, aggrb=160693KB/s, minb=160693KB/s, maxb=160693KB/s, mint=815665msec, maxt=815665msec
warm: READ: io= 81920MB, aggrb=109853KB/s, minb= 27463KB/s, maxb= 29244KB/s, mint=717110msec, maxt=763617msec
cold: READ: io= 30720MB, aggrb= 35245KB/s, minb= 35245KB/s, maxb= 35245KB/s, mint=892530msec, maxt=892530msec
sdb: ios=797960/4, merge=11763/1, ticks=4307910/796, in_queue=4308380, util=100.00%
patched:
hot: READ: io=128000MB, aggrb=160678KB/s, minb=160678KB/s, maxb=160678KB/s, mint=815740msec, maxt=815740msec
warm: READ: io= 81920MB, aggrb=147747KB/s, minb= 36936KB/s, maxb= 40960KB/s, mint=512000msec, maxt=567767msec
cold: READ: io= 30720MB, aggrb= 40960KB/s, minb= 40960KB/s, maxb= 40960KB/s, mint=768000msec, maxt=768000msec
sdb: ios=596514/4, merge=9341/1, ticks=2395362/997, in_queue=2396484, util=79.18%
In both kernels, the hot set is propagated to the active list and then
served from cache.
In both kernels, the beginning of the warm set is propagated to the
active list as well, but in the unpatched case the active list
eventually takes up half of memory and no new pages from the warm set
get activated, despite repeated access, and despite most of the active
list soon being stale. The patched kernel on the other hand detects the
thrashing and manages to keep this cache window rolling through the data
set. This frees up enough IO bandwidth that the cold set is served at
full speed as well and disk utilization even drops by 20%.
For reference, this same test was performed with the traditional
demotion mechanism, where deactivation is coupled to inactive list
reclaim. However, this had the same outcome as the unpatched kernel:
while the warm set does indeed get activated continuously, it is forced
out of the active list by inactive list pressure, which is dictated
primarily by the unrelated cold set. The warm set is evicted before
subsequent streamers can benefit from it, even though there would be
enough space available to cache the pages of interest.
Costs:
Page reclaim used to shrink the radix trees but now the tree nodes are
reused for shadow entries, where the cost depends heavily on the page
cache access patterns. However, with workloads that maintain spatial or
temporal locality, the shadow entries are either refaulted quickly or
reclaimed along with the inode object itself. Workloads that will
experience a memory cost increase are those that don't really benefit
from caching in the first place.
A more predictable alternative would be a fixed-cost separate pool of
shadow entries, but this would incur relatively higher memory cost for
well-behaved workloads at the benefit of cornercases. It would also
make the shadow entry lookup more costly compared to storing them
directly in the cache structure.
Future:
To simplify the merging process, this patch set is implementing thrash
detection on a global per-zone level only for now, but the design is
such that it can be extended to memory cgroups as well. All we need to
do is store the unique cgroup ID along the node and zone identifier
inside the eviction cookie to identify the lruvec.
Right now we have a fixed ratio (50:50) between inactive and active list
but we already have complaints about working sets exceeding half of
memory being pushed out of the cache by simple streaming in the
background. Ultimately, we want to adjust this ratio and allow for a
much smaller inactive list. These patches are an essential step in this
direction because they decouple the VMs ability to detect working set
changes from the inactive list size. This would allow us to base the
inactive list size on the combined readahead window size for example and
potentially protect a much bigger working set.
It's also a big step towards activating pages with a reuse distance
larger than memory, as long as they are the most frequently used pages
in the workload. This will require knowing more about the access
frequency of active pages than what we measure right now, so it's also
deferred in this series.
Another possibility of having thrashing information would be to revisit
the idea of local reclaim in the form of zero-config memory control
groups. Instead of having allocating tasks go straight to global
reclaim, they could try to reclaim the pages in the memcg they are part
of first as long as the group is not thrashing. This would allow a user
to drop e.g. a back-up job in an otherwise unconfigured memcg and it
would only inflate (and possibly do global reclaim) until it has enough
memory to do proper readahead. But once it reaches that point and stops
thrashing it would just recycle its own used-once pages without kicking
out the cache of any other tasks in the system more than necessary.
This patch (of 10):
Fengguang Wu's build testing spotted problems with inc_zone_state() and
dec_zone_state() on UP configurations in out-of-tree patches.
inc_zone_state() is declared but not defined, dec_zone_state() is
missing entirely.
Just like with *_zone_page_state(), they can be defined like their
preemption-unsafe counterparts on UP.
[akpm@linux-foundation.org: make it build]
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Bob Liu <bob.liu@oracle.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Luigi Semenzato <semenzato@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Metin Doslu <metin@citusdata.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Ozgun Erdogan <ozgun@citusdata.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Roman Gushchin <klamm@yandex-team.ru>
Cc: Ryan Mallon <rmallon@gmail.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-04 05:47:34 +08:00
|
|
|
static inline void __inc_zone_page_state(struct page *page,
|
|
|
|
|
enum zone_stat_item item)
|
|
|
|
|
{
|
|
|
|
|
__inc_zone_state(page_zone(page), item);
|
|
|
|
|
}
|
|
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
static inline void __dec_zone_page_state(struct page *page,
|
|
|
|
|
enum zone_stat_item item)
|
|
|
|
|
{
|
2008-02-25 23:45:03 +08:00
|
|
|
__dec_zone_state(page_zone(page), item);
|
2006-06-30 16:55:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We only use atomic operations to update counters. So there is no need to
|
|
|
|
|
* disable interrupts.
|
|
|
|
|
*/
|
|
|
|
|
#define inc_zone_page_state __inc_zone_page_state
|
|
|
|
|
#define dec_zone_page_state __dec_zone_page_state
|
|
|
|
|
#define mod_zone_page_state __mod_zone_page_state
|
|
|
|
|
|
mm: vmstat: fix UP zone state accounting
Summary:
The VM maintains cached filesystem pages on two types of lists. One
list holds the pages recently faulted into the cache, the other list
holds pages that have been referenced repeatedly on that first list.
The idea is to prefer reclaiming young pages over those that have shown
to benefit from caching in the past. We call the recently used list
"inactive list" and the frequently used list "active list".
Currently, the VM aims for a 1:1 ratio between the lists, which is the
"perfect" trade-off between the ability to *protect* frequently used
pages and the ability to *detect* frequently used pages. This means
that working set changes bigger than half of cache memory go undetected
and thrash indefinitely, whereas working sets bigger than half of cache
memory are unprotected against used-once streams that don't even need
caching.
This happens on file servers and media streaming servers, where the
popular files and file sections change over time. Even though the
individual files might be smaller than half of memory, concurrent access
to many of them may still result in their inter-reference distance being
greater than half of memory. It's also been reported as a problem on
database workloads that switch back and forth between tables that are
bigger than half of memory. In these cases the VM never recognizes the
new working set and will for the remainder of the workload thrash disk
data which could easily live in memory.
Historically, every reclaim scan of the inactive list also took a
smaller number of pages from the tail of the active list and moved them
to the head of the inactive list. This model gave established working
sets more gracetime in the face of temporary use-once streams, but
ultimately was not significantly better than a FIFO policy and still
thrashed cache based on eviction speed, rather than actual demand for
cache.
This series solves the problem by maintaining a history of pages evicted
from the inactive list, enabling the VM to detect frequently used pages
regardless of inactive list size and facilitate working set transitions.
Tests:
The reported database workload is easily demonstrated on a 8G machine
with two filesets a 6G. This fio workload operates on one set first,
then switches to the other. The VM should obviously always cache the
set that the workload is currently using.
This test is based on a problem encountered by Citus Data customers:
http://citusdata.com/blog/72-linux-memory-manager-and-your-big-data
unpatched:
db1: READ: io=98304MB, aggrb=885559KB/s, minb=885559KB/s, maxb=885559KB/s, mint= 113672msec, maxt= 113672msec
db2: READ: io=98304MB, aggrb= 66169KB/s, minb= 66169KB/s, maxb= 66169KB/s, mint=1521302msec, maxt=1521302msec
sdb: ios=835750/4, merge=2/1, ticks=4659739/60016, in_queue=4719203, util=98.92%
real 27m15.541s
user 0m19.059s
sys 0m51.459s
patched:
db1: READ: io=98304MB, aggrb=877783KB/s, minb=877783KB/s, maxb=877783KB/s, mint=114679msec, maxt=114679msec
db2: READ: io=98304MB, aggrb=397449KB/s, minb=397449KB/s, maxb=397449KB/s, mint=253273msec, maxt=253273msec
sdb: ios=170587/4, merge=2/1, ticks=954910/61123, in_queue=1015923, util=90.40%
real 6m8.630s
user 0m14.714s
sys 0m31.233s
As can be seen, the unpatched kernel simply never adapts to the
workingset change and db2 is stuck indefinitely with secondary storage
speed. The patched kernel needs 2-3 iterations over db2 before it
replaces db1 and reaches full memory speed. Given the unbounded
negative affect of the existing VM behavior, these patches should be
considered correctness fixes rather than performance optimizations.
Another test resembles a fileserver or streaming server workload, where
data in excess of memory size is accessed at different frequencies.
There is very hot data accessed at a high frequency. Machines should be
fitted so that the hot set of such a workload can be fully cached or all
bets are off. Then there is a very big (compared to available memory)
set of data that is used-once or at a very low frequency; this is what
drives the inactive list and does not really benefit from caching.
Lastly, there is a big set of warm data in between that is accessed at
medium frequencies and benefits from caching the pages between the first
and last streamer of each burst.
unpatched:
hot: READ: io=128000MB, aggrb=160693KB/s, minb=160693KB/s, maxb=160693KB/s, mint=815665msec, maxt=815665msec
warm: READ: io= 81920MB, aggrb=109853KB/s, minb= 27463KB/s, maxb= 29244KB/s, mint=717110msec, maxt=763617msec
cold: READ: io= 30720MB, aggrb= 35245KB/s, minb= 35245KB/s, maxb= 35245KB/s, mint=892530msec, maxt=892530msec
sdb: ios=797960/4, merge=11763/1, ticks=4307910/796, in_queue=4308380, util=100.00%
patched:
hot: READ: io=128000MB, aggrb=160678KB/s, minb=160678KB/s, maxb=160678KB/s, mint=815740msec, maxt=815740msec
warm: READ: io= 81920MB, aggrb=147747KB/s, minb= 36936KB/s, maxb= 40960KB/s, mint=512000msec, maxt=567767msec
cold: READ: io= 30720MB, aggrb= 40960KB/s, minb= 40960KB/s, maxb= 40960KB/s, mint=768000msec, maxt=768000msec
sdb: ios=596514/4, merge=9341/1, ticks=2395362/997, in_queue=2396484, util=79.18%
In both kernels, the hot set is propagated to the active list and then
served from cache.
In both kernels, the beginning of the warm set is propagated to the
active list as well, but in the unpatched case the active list
eventually takes up half of memory and no new pages from the warm set
get activated, despite repeated access, and despite most of the active
list soon being stale. The patched kernel on the other hand detects the
thrashing and manages to keep this cache window rolling through the data
set. This frees up enough IO bandwidth that the cold set is served at
full speed as well and disk utilization even drops by 20%.
For reference, this same test was performed with the traditional
demotion mechanism, where deactivation is coupled to inactive list
reclaim. However, this had the same outcome as the unpatched kernel:
while the warm set does indeed get activated continuously, it is forced
out of the active list by inactive list pressure, which is dictated
primarily by the unrelated cold set. The warm set is evicted before
subsequent streamers can benefit from it, even though there would be
enough space available to cache the pages of interest.
Costs:
Page reclaim used to shrink the radix trees but now the tree nodes are
reused for shadow entries, where the cost depends heavily on the page
cache access patterns. However, with workloads that maintain spatial or
temporal locality, the shadow entries are either refaulted quickly or
reclaimed along with the inode object itself. Workloads that will
experience a memory cost increase are those that don't really benefit
from caching in the first place.
A more predictable alternative would be a fixed-cost separate pool of
shadow entries, but this would incur relatively higher memory cost for
well-behaved workloads at the benefit of cornercases. It would also
make the shadow entry lookup more costly compared to storing them
directly in the cache structure.
Future:
To simplify the merging process, this patch set is implementing thrash
detection on a global per-zone level only for now, but the design is
such that it can be extended to memory cgroups as well. All we need to
do is store the unique cgroup ID along the node and zone identifier
inside the eviction cookie to identify the lruvec.
Right now we have a fixed ratio (50:50) between inactive and active list
but we already have complaints about working sets exceeding half of
memory being pushed out of the cache by simple streaming in the
background. Ultimately, we want to adjust this ratio and allow for a
much smaller inactive list. These patches are an essential step in this
direction because they decouple the VMs ability to detect working set
changes from the inactive list size. This would allow us to base the
inactive list size on the combined readahead window size for example and
potentially protect a much bigger working set.
It's also a big step towards activating pages with a reuse distance
larger than memory, as long as they are the most frequently used pages
in the workload. This will require knowing more about the access
frequency of active pages than what we measure right now, so it's also
deferred in this series.
Another possibility of having thrashing information would be to revisit
the idea of local reclaim in the form of zero-config memory control
groups. Instead of having allocating tasks go straight to global
reclaim, they could try to reclaim the pages in the memcg they are part
of first as long as the group is not thrashing. This would allow a user
to drop e.g. a back-up job in an otherwise unconfigured memcg and it
would only inflate (and possibly do global reclaim) until it has enough
memory to do proper readahead. But once it reaches that point and stops
thrashing it would just recycle its own used-once pages without kicking
out the cache of any other tasks in the system more than necessary.
This patch (of 10):
Fengguang Wu's build testing spotted problems with inc_zone_state() and
dec_zone_state() on UP configurations in out-of-tree patches.
inc_zone_state() is declared but not defined, dec_zone_state() is
missing entirely.
Just like with *_zone_page_state(), they can be defined like their
preemption-unsafe counterparts on UP.
[akpm@linux-foundation.org: make it build]
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Bob Liu <bob.liu@oracle.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Luigi Semenzato <semenzato@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Metin Doslu <metin@citusdata.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Ozgun Erdogan <ozgun@citusdata.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Roman Gushchin <klamm@yandex-team.ru>
Cc: Ryan Mallon <rmallon@gmail.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-04 05:47:34 +08:00
|
|
|
#define inc_zone_state __inc_zone_state
|
|
|
|
|
#define dec_zone_state __dec_zone_state
|
|
|
|
|
|
2011-01-14 07:45:43 +08:00
|
|
|
#define set_pgdat_percpu_threshold(pgdat, callback) { }
|
mm: page allocator: adjust the per-cpu counter threshold when memory is low
Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory
is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To
avoid synchronization overhead, these counters are maintained on a per-cpu
basis and drained both periodically and when a threshold is above a
threshold. On large CPU systems, the difference between the estimate and
real value of NR_FREE_PAGES can be very high. The system can get into a
case where pages are allocated far below the min watermark potentially
causing livelock issues. The commit solved the problem by taking a better
reading of NR_FREE_PAGES when memory was low.
Unfortately, as reported by Shaohua Li this accurate reading can consume a
large amount of CPU time on systems with many sockets due to cache line
bouncing. This patch takes a different approach. For large machines
where counter drift might be unsafe and while kswapd is awake, the per-cpu
thresholds for the target pgdat are reduced to limit the level of drift to
what should be a safe level. This incurs a performance penalty in heavy
memory pressure by a factor that depends on the workload and the machine
but the machine should function correctly without accidentally exhausting
all memory on a node. There is an additional cost when kswapd wakes and
sleeps but the event is not expected to be frequent - in Shaohua's test
case, there was one recorded sleep and wake event at least.
To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is
introduced that takes a more accurate reading of NR_FREE_PAGES when called
from wakeup_kswapd, when deciding whether it is really safe to go back to
sleep in sleeping_prematurely() and when deciding if a zone is really
balanced or not in balance_pgdat(). We are still using an expensive
function but limiting how often it is called.
When the test case is reproduced, the time spent in the watermark
functions is reduced. The following report is on the percentage of time
spent cumulatively spent in the functions zone_nr_free_pages(),
zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(),
zone_page_state_snapshot(), zone_page_state().
vanilla 11.6615%
disable-threshold 0.2584%
David said:
: We had to pull aa454840 "mm: page allocator: calculate a better estimate
: of NR_FREE_PAGES when memory is low and kswapd is awake" from 2.6.36
: internally because tests showed that it would cause the machine to stall
: as the result of heavy kswapd activity. I merged it back with this fix as
: it is pending in the -mm tree and it solves the issue we were seeing, so I
: definitely think this should be pushed to -stable (and I would seriously
: consider it for 2.6.37 inclusion even at this late date).
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Reported-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Tested-by: Nicolas Bareil <nico@chdir.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: <stable@kernel.org> [2.6.37.1, 2.6.36.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-14 07:45:41 +08:00
|
|
|
|
2006-06-30 16:55:33 +08:00
|
|
|
static inline void refresh_cpu_vm_stats(int cpu) { }
|
2011-05-25 08:11:33 +08:00
|
|
|
static inline void refresh_zone_stat_thresholds(void) { }
|
2013-09-12 05:21:30 +08:00
|
|
|
static inline void cpu_vm_stats_fold(int cpu) { }
|
2011-05-25 08:11:33 +08:00
|
|
|
|
2012-10-09 07:33:39 +08:00
|
|
|
static inline void drain_zonestat(struct zone *zone,
|
|
|
|
|
struct per_cpu_pageset *pset) { }
|
2011-05-25 08:11:28 +08:00
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
|
2012-10-09 07:32:02 +08:00
|
|
|
static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
|
|
|
|
|
int migratetype)
|
|
|
|
|
{
|
|
|
|
|
__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
|
|
|
|
|
if (is_migrate_cma(migratetype))
|
|
|
|
|
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
|
|
|
|
|
}
|
|
|
|
|
|
2011-05-25 08:11:28 +08:00
|
|
|
extern const char * const vmstat_text[];
|
2006-06-30 16:55:33 +08:00
|
|
|
|
|
|
|
|
#endif /* _LINUX_VMSTAT_H */
|