hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mm: multi-gen LRU: rename lrugen->lists[] to lrugen->folios[] 

commit 6df1b22129 upstream.

lru_gen_folio will be chained into per-node lists by the coming
lrugen->list.

Link: https://lkml.kernel.org/r/20221222041905.2431096-3-yuzhao@google.com
Signed-off-by: Yu Zhao <yuzhao@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Michael Larabel <Michael@MichaelLarabel.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
Yu Zhao 2022-12-21 21:19:00 -07:00 committed by Greg Kroah-Hartman
parent 7164d74aae
commit a73d04c460
4 changed files with 20 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Documentation/mm/multigen_lru.rst
View File

@ -89,15 +89,15 @@ variables are monotonically increasing.
Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)``
bits in order to fit into the gen counter in ``folio->flags``. Each bits in order to fit into the gen counter in ``folio->flags``. Each
truncated generation number is an index to ``lrugen->lists[]``. The truncated generation number is an index to ``lrugen->folios[]``. The
sliding window technique is used to track at least ``MIN_NR_GENS`` and sliding window technique is used to track at least ``MIN_NR_GENS`` and
at most ``MAX_NR_GENS`` generations. The gen counter stores a value at most ``MAX_NR_GENS`` generations. The gen counter stores a value
within ``[1, MAX_NR_GENS]`` while a page is on one of within ``[1, MAX_NR_GENS]`` while a page is on one of
``lrugen->lists[]``; otherwise it stores zero. ``lrugen->folios[]``; otherwise it stores zero.
Each generation is divided into multiple tiers. A page accessed ``N`` Each generation is divided into multiple tiers. A page accessed ``N``
times through file descriptors is in tier ``order_base_2(N)``. Unlike times through file descriptors is in tier ``order_base_2(N)``. Unlike
generations, tiers do not have dedicated ``lrugen->lists[]``. In generations, tiers do not have dedicated ``lrugen->folios[]``. In
contrast to moving across generations, which requires the LRU lock, contrast to moving across generations, which requires the LRU lock,
moving across tiers only involves atomic operations on moving across tiers only involves atomic operations on
``folio->flags`` and therefore has a negligible cost. A feedback loop ``folio->flags`` and therefore has a negligible cost. A feedback loop
@ -127,7 +127,7 @@ page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``.
Eviction Eviction
-------- --------
The eviction consumes old generations. Given an ``lruvec``, it The eviction consumes old generations. Given an ``lruvec``, it
increments ``min_seq`` when ``lrugen->lists[]`` indexed by increments ``min_seq`` when ``lrugen->folios[]`` indexed by
``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to ``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to
evict from, it first compares ``min_seq[]`` to select the older type. evict from, it first compares ``min_seq[]`` to select the older type.
If both types are equally old, it selects the one whose first tier has If both types are equally old, it selects the one whose first tier has

4
include/linux/mm_inline.h
View File

@ -256,9 +256,9 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
lru_gen_update_size(lruvec, folio, -1, gen); lru_gen_update_size(lruvec, folio, -1, gen);
/* for folio_rotate_reclaimable() */ /* for folio_rotate_reclaimable() */
if (reclaiming) if (reclaiming)
list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
else else
list_add(&folio->lru, &lrugen->lists[gen][type][zone]); list_add(&folio->lru, &lrugen->folios[gen][type][zone]);
return true; return true;
} }

8
include/linux/mmzone.h
View File

@ -312,7 +312,7 @@ enum lruvec_flags {
* They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An
* offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the
* corresponding generation. The gen counter in folio->flags stores gen+1 while * corresponding generation. The gen counter in folio->flags stores gen+1 while
* a page is on one of lrugen->lists[]. Otherwise it stores 0. * a page is on one of lrugen->folios[]. Otherwise it stores 0.
* *
* A page is added to the youngest generation on faulting. The aging needs to * A page is added to the youngest generation on faulting. The aging needs to
* check the accessed bit at least twice before handing this page over to the * check the accessed bit at least twice before handing this page over to the
@ -324,8 +324,8 @@ enum lruvec_flags {
* rest of generations, if they exist, are considered inactive. See * rest of generations, if they exist, are considered inactive. See
* lru_gen_is_active(). * lru_gen_is_active().
* *
* PG_active is always cleared while a page is on one of lrugen->lists[] so that * PG_active is always cleared while a page is on one of lrugen->folios[] so
* the aging needs not to worry about it. And it's set again when a page * that the aging needs not to worry about it. And it's set again when a page
* considered active is isolated for non-reclaiming purposes, e.g., migration. * considered active is isolated for non-reclaiming purposes, e.g., migration.
* See lru_gen_add_folio() and lru_gen_del_folio(). * See lru_gen_add_folio() and lru_gen_del_folio().
* *
@ -412,7 +412,7 @@ struct lru_gen_struct {
/* the birth time of each generation in jiffies */ /* the birth time of each generation in jiffies */
unsigned long timestamps[MAX_NR_GENS]; unsigned long timestamps[MAX_NR_GENS];
/* the multi-gen LRU lists, lazily sorted on eviction */ /* the multi-gen LRU lists, lazily sorted on eviction */
struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; struct list_head folios[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the multi-gen LRU sizes, eventually consistent */ /* the multi-gen LRU sizes, eventually consistent */
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */ /* the exponential moving average of refaulted */

20
mm/vmscan.c
View File

@ -4258,7 +4258,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
/* prevent cold/hot inversion if force_scan is true */ /* prevent cold/hot inversion if force_scan is true */
for (zone = 0; zone < MAX_NR_ZONES; zone++) { for (zone = 0; zone < MAX_NR_ZONES; zone++) {
struct list_head *head = &lrugen->lists[old_gen][type][zone]; struct list_head *head = &lrugen->folios[old_gen][type][zone];
while (!list_empty(head)) { while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head); struct folio *folio = lru_to_folio(head);
@ -4269,7 +4269,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap)
VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
new_gen = folio_inc_gen(lruvec, folio, false); new_gen = folio_inc_gen(lruvec, folio, false);
list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
if (!--remaining) if (!--remaining)
return false; return false;
@ -4297,7 +4297,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
gen = lru_gen_from_seq(min_seq[type]); gen = lru_gen_from_seq(min_seq[type]);
for (zone = 0; zone < MAX_NR_ZONES; zone++) { for (zone = 0; zone < MAX_NR_ZONES; zone++) {
if (!list_empty(&lrugen->lists[gen][type][zone])) if (!list_empty(&lrugen->folios[gen][type][zone]))
goto next; goto next;
} }
@ -4762,7 +4762,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
/* promoted */ /* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
list_move(&folio->lru, &lrugen->lists[gen][type][zone]); list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
return true; return true;
} }
@ -4771,7 +4771,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
int hist = lru_hist_from_seq(lrugen->min_seq[type]); int hist = lru_hist_from_seq(lrugen->min_seq[type]);
gen = folio_inc_gen(lruvec, folio, false); gen = folio_inc_gen(lruvec, folio, false);
list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
WRITE_ONCE(lrugen->protected[hist][type][tier - 1], WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
lrugen->protected[hist][type][tier - 1] + delta); lrugen->protected[hist][type][tier - 1] + delta);
@ -4783,7 +4783,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx)
if (folio_test_locked(folio) || folio_test_writeback(folio) || if (folio_test_locked(folio) || folio_test_writeback(folio) ||
(type == LRU_GEN_FILE && folio_test_dirty(folio))) { (type == LRU_GEN_FILE && folio_test_dirty(folio))) {
gen = folio_inc_gen(lruvec, folio, true); gen = folio_inc_gen(lruvec, folio, true);
list_move(&folio->lru, &lrugen->lists[gen][type][zone]); list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
return true; return true;
} }
@ -4850,7 +4850,7 @@ static int scan_folios(struct lruvec *lruvec, struct scan_control *sc,
for (zone = sc->reclaim_idx; zone >= 0; zone--) { for (zone = sc->reclaim_idx; zone >= 0; zone--) {
LIST_HEAD(moved); LIST_HEAD(moved);
int skipped = 0; int skipped = 0;
struct list_head *head = &lrugen->lists[gen][type][zone]; struct list_head *head = &lrugen->folios[gen][type][zone];
while (!list_empty(head)) { while (!list_empty(head)) {
struct folio *folio = lru_to_folio(head); struct folio *folio = lru_to_folio(head);
@ -5250,7 +5250,7 @@ static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
int gen, type, zone; int gen, type, zone;
for_each_gen_type_zone(gen, type, zone) { for_each_gen_type_zone(gen, type, zone) {
if (!list_empty(&lrugen->lists[gen][type][zone])) if (!list_empty(&lrugen->folios[gen][type][zone]))
return false; return false;
} }
} }
@ -5295,7 +5295,7 @@ static bool drain_evictable(struct lruvec *lruvec)
int remaining = MAX_LRU_BATCH; int remaining = MAX_LRU_BATCH;
for_each_gen_type_zone(gen, type, zone) { for_each_gen_type_zone(gen, type, zone) {
struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
while (!list_empty(head)) { while (!list_empty(head)) {
bool success; bool success;
@ -5832,7 +5832,7 @@ void lru_gen_init_lruvec(struct lruvec *lruvec)
lrugen->timestamps[i] = jiffies; lrugen->timestamps[i] = jiffies;
for_each_gen_type_zone(gen, type, zone) for_each_gen_type_zone(gen, type, zone)
INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]);
lruvec->mm_state.seq = MIN_NR_GENS; lruvec->mm_state.seq = MIN_NR_GENS;
init_waitqueue_head(&lruvec->mm_state.wait); init_waitqueue_head(&lruvec->mm_state.wait);