UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

migrate: convert migrate_pages() to use folios 

Quite straightforward, the page functions are converted to corresponding
folio functions.  Same for comments.

THP specific code are converted to be large folio.

Link: https://lkml.kernel.org/r/20221109012348.93849-3-ying.huang@intel.com
Signed-off-by: "Huang, Ying" <ying.huang@intel.com>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Tested-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Matthew Wilcox <willy@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Huang Ying 2022-11-09 09:23:48 +08:00 committed by Andrew Morton
parent 49f5185922
commit eaec4e639f
1 changed files with 112 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
mm/migrate.c
View File

@ -1373,231 +1373,245 @@ out:
return rc; return rc;
} }
static inline int try_split_thp(struct page *page, struct list_head *split_pages) static inline int try_split_folio(struct folio *folio, struct list_head *split_folios)
{ {
int rc; int rc;
lock_page(page); folio_lock(folio);
rc = split_huge_page_to_list(page, split_pages); rc = split_folio_to_list(folio, split_folios);
unlock_page(page); folio_unlock(folio);
if (!rc) if (!rc)
list_move_tail(&page->lru, split_pages); list_move_tail(&folio->lru, split_folios);
return rc; return rc;
} }
/* /*
* migrate_pages - migrate the pages specified in a list, to the free pages * migrate_pages - migrate the folios specified in a list, to the free folios
* supplied as the target for the page migration * supplied as the target for the page migration
* *
* @from: The list of pages to be migrated. * @from: The list of folios to be migrated.
* @get_new_page: The function used to allocate free pages to be used * @get_new_page: The function used to allocate free folios to be used
* as the target of the page migration. * as the target of the folio migration.
* @put_new_page: The function used to free target pages if migration * @put_new_page: The function used to free target folios if migration
* fails, or NULL if no special handling is necessary. * fails, or NULL if no special handling is necessary.
* @private: Private data to be passed on to get_new_page() * @private: Private data to be passed on to get_new_page()
* @mode: The migration mode that specifies the constraints for * @mode: The migration mode that specifies the constraints for
* page migration, if any. * folio migration, if any.
* @reason: The reason for page migration. * @reason: The reason for folio migration.
* @ret_succeeded: Set to the number of normal pages migrated successfully if * @ret_succeeded: Set to the number of folios migrated successfully if
* the caller passes a non-NULL pointer. * the caller passes a non-NULL pointer.
* *
* The function returns after 10 attempts or if no pages are movable any more * The function returns after 10 attempts or if no folios are movable any more
* because the list has become empty or no retryable pages exist any more. * because the list has become empty or no retryable folios exist any more.
* It is caller's responsibility to call putback_movable_pages() to return pages * It is caller's responsibility to call putback_movable_pages() to return folios
* to the LRU or free list only if ret != 0. * to the LRU or free list only if ret != 0.
* *
* Returns the number of {normal page, THP, hugetlb} that were not migrated, or * Returns the number of {normal folio, large folio, hugetlb} that were not
* an error code. The number of THP splits will be considered as the number of * migrated, or an error code. The number of large folio splits will be
* non-migrated THP, no matter how many subpages of the THP are migrated successfully. * considered as the number of non-migrated large folio, no matter how many
* split folios of the large folio are migrated successfully.
*/ */
int migrate_pages(struct list_head *from, new_page_t get_new_page, int migrate_pages(struct list_head *from, new_page_t get_new_page,
free_page_t put_new_page, unsigned long private, free_page_t put_new_page, unsigned long private,
enum migrate_mode mode, int reason, unsigned int *ret_succeeded) enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
{ {
int retry = 1; int retry = 1;
int large_retry = 1;
int thp_retry = 1; int thp_retry = 1;
int nr_failed = 0; int nr_failed = 0;
int nr_failed_pages = 0; int nr_failed_pages = 0;
int nr_retry_pages = 0; int nr_retry_pages = 0;
int nr_succeeded = 0; int nr_succeeded = 0;
int nr_thp_succeeded = 0; int nr_thp_succeeded = 0;
int nr_large_failed = 0;
int nr_thp_failed = 0; int nr_thp_failed = 0;
int nr_thp_split = 0; int nr_thp_split = 0;
int pass = 0; int pass = 0;
bool is_large = false;
bool is_thp = false; bool is_thp = false;
struct page *page; struct folio *folio, *folio2;
struct page *page2; int rc, nr_pages;
int rc, nr_subpages; LIST_HEAD(ret_folios);
LIST_HEAD(ret_pages); LIST_HEAD(split_folios);
LIST_HEAD(thp_split_pages);
bool nosplit = (reason == MR_NUMA_MISPLACED); bool nosplit = (reason == MR_NUMA_MISPLACED);
bool no_subpage_counting = false; bool no_split_folio_counting = false;
trace_mm_migrate_pages_start(mode, reason); trace_mm_migrate_pages_start(mode, reason);
thp_subpage_migration: split_folio_migration:
for (pass = 0; pass < 10 && (retry || thp_retry); pass++) { for (pass = 0; pass < 10 && (retry || large_retry); pass++) {
retry = 0; retry = 0;
large_retry = 0;
thp_retry = 0; thp_retry = 0;
nr_retry_pages = 0; nr_retry_pages = 0;
list_for_each_entry_safe(page, page2, from, lru) { list_for_each_entry_safe(folio, folio2, from, lru) {
/* /*
* THP statistics is based on the source huge page. * Large folio statistics is based on the source large
* Capture required information that might get lost * folio. Capture required information that might get
* during migration. * lost during migration.
*/ */
is_thp = PageTransHuge(page) && !PageHuge(page); is_large = folio_test_large(folio) && !folio_test_hugetlb(folio);
nr_subpages = compound_nr(page); is_thp = is_large && folio_test_pmd_mappable(folio);
nr_pages = folio_nr_pages(folio);
cond_resched(); cond_resched();
if (PageHuge(page)) if (folio_test_hugetlb(folio))
rc = unmap_and_move_huge_page(get_new_page, rc = unmap_and_move_huge_page(get_new_page,
put_new_page, private, page, put_new_page, private,
pass > 2, mode, reason, &folio->page, pass > 2, mode,
&ret_pages); reason,
&ret_folios);
else else
rc = unmap_and_move(get_new_page, put_new_page, rc = unmap_and_move(get_new_page, put_new_page,
private, page_folio(page), pass > 2, mode, private, folio, pass > 2, mode,
reason, &ret_pages); reason, &ret_folios);
/* /*
* The rules are: * The rules are:
* Success: non hugetlb page will be freed, hugetlb * Success: non hugetlb folio will be freed, hugetlb
* page will be put back * folio will be put back
* -EAGAIN: stay on the from list * -EAGAIN: stay on the from list
* -ENOMEM: stay on the from list * -ENOMEM: stay on the from list
* -ENOSYS: stay on the from list * -ENOSYS: stay on the from list
* Other errno: put on ret_pages list then splice to * Other errno: put on ret_folios list then splice to
* from list * from list
*/ */
switch(rc) { switch(rc) {
/* /*
* THP migration might be unsupported or the * Large folio migration might be unsupported or
* allocation could've failed so we should * the allocation could've failed so we should retry
* retry on the same page with the THP split * on the same folio with the large folio split
* to base pages. * to normal folios.
* *
* Sub-pages are put in thp_split_pages, and * Split folios are put in split_folios, and
* we will migrate them after the rest of the * we will migrate them after the rest of the
* list is processed. * list is processed.
*/ */
case -ENOSYS: case -ENOSYS:
/* THP migration is unsupported */ /* Large folio migration is unsupported */
if (is_thp) { if (is_large) {
nr_thp_failed++; nr_large_failed++;
if (!try_split_thp(page, &thp_split_pages)) { nr_thp_failed += is_thp;
nr_thp_split++; if (!try_split_folio(folio, &split_folios)) {
nr_thp_split += is_thp;
break; break;
} }
/* Hugetlb migration is unsupported */ /* Hugetlb migration is unsupported */
} else if (!no_subpage_counting) { } else if (!no_split_folio_counting) {
nr_failed++; nr_failed++;
} }
nr_failed_pages += nr_subpages; nr_failed_pages += nr_pages;
list_move_tail(&page->lru, &ret_pages); list_move_tail(&folio->lru, &ret_folios);
break; break;
case -ENOMEM: case -ENOMEM:
/* /*
* When memory is low, don't bother to try to migrate * When memory is low, don't bother to try to migrate
* other pages, just exit. * other folios, just exit.
*/ */
if (is_thp) { if (is_large) {
nr_thp_failed++; nr_large_failed++;
/* THP NUMA faulting doesn't split THP to retry. */ nr_thp_failed += is_thp;
/* Large folio NUMA faulting doesn't split to retry. */
if (!nosplit) { if (!nosplit) {
int ret = try_split_thp(page, &thp_split_pages); int ret = try_split_folio(folio, &split_folios);
if (!ret) { if (!ret) {
nr_thp_split++; nr_thp_split += is_thp;
break; break;
} else if (reason == MR_LONGTERM_PIN && } else if (reason == MR_LONGTERM_PIN &&
ret == -EAGAIN) { ret == -EAGAIN) {
/* /*
* Try again to split THP to mitigate * Try again to split large folio to
* the failure of longterm pinning. * mitigate the failure of longterm pinning.
*/ */
thp_retry++; large_retry++;
nr_retry_pages += nr_subpages; thp_retry += is_thp;
nr_retry_pages += nr_pages;
break; break;
} }
} }
} else if (!no_subpage_counting) { } else if (!no_split_folio_counting) {
nr_failed++; nr_failed++;
} }
nr_failed_pages += nr_subpages + nr_retry_pages; nr_failed_pages += nr_pages + nr_retry_pages;
/* /*
* There might be some subpages of fail-to-migrate THPs * There might be some split folios of fail-to-migrate large
* left in thp_split_pages list. Move them back to migration * folios left in split_folios list. Move them back to migration
* list so that they could be put back to the right list by * list so that they could be put back to the right list by
* the caller otherwise the page refcnt will be leaked. * the caller otherwise the folio refcnt will be leaked.
*/ */
list_splice_init(&thp_split_pages, from); list_splice_init(&split_folios, from);
/* nr_failed isn't updated for not used */ /* nr_failed isn't updated for not used */
nr_large_failed += large_retry;
nr_thp_failed += thp_retry; nr_thp_failed += thp_retry;
goto out; goto out;
case -EAGAIN: case -EAGAIN:
if (is_thp) if (is_large) {
thp_retry++; large_retry++;
else if (!no_subpage_counting) thp_retry += is_thp;
} else if (!no_split_folio_counting) {
retry++; retry++;
nr_retry_pages += nr_subpages; }
nr_retry_pages += nr_pages;
break; break;
case MIGRATEPAGE_SUCCESS: case MIGRATEPAGE_SUCCESS:
nr_succeeded += nr_subpages; nr_succeeded += nr_pages;
if (is_thp) nr_thp_succeeded += is_thp;
nr_thp_succeeded++;
break; break;
default: default:
/* /*
* Permanent failure (-EBUSY, etc.): * Permanent failure (-EBUSY, etc.):
* unlike -EAGAIN case, the failed page is * unlike -EAGAIN case, the failed folio is
* removed from migration page list and not * removed from migration folio list and not
* retried in the next outer loop. * retried in the next outer loop.
*/ */
if (is_thp) if (is_large) {
nr_thp_failed++; nr_large_failed++;
else if (!no_subpage_counting) nr_thp_failed += is_thp;
} else if (!no_split_folio_counting) {
nr_failed++; nr_failed++;
}
nr_failed_pages += nr_subpages; nr_failed_pages += nr_pages;
break; break;
} }
} }
} }
nr_failed += retry; nr_failed += retry;
nr_large_failed += large_retry;
nr_thp_failed += thp_retry; nr_thp_failed += thp_retry;
nr_failed_pages += nr_retry_pages; nr_failed_pages += nr_retry_pages;
/* /*
* Try to migrate subpages of fail-to-migrate THPs, no nr_failed * Try to migrate split folios of fail-to-migrate large folios, no
* counting in this round, since all subpages of a THP is counted * nr_failed counting in this round, since all split folios of a
* as 1 failure in the first round. * large folio is counted as 1 failure in the first round.
*/ */
if (!list_empty(&thp_split_pages)) { if (!list_empty(&split_folios)) {
/* /*
* Move non-migrated pages (after 10 retries) to ret_pages * Move non-migrated folios (after 10 retries) to ret_folios
* to avoid migrating them again. * to avoid migrating them again.
*/ */
list_splice_init(from, &ret_pages); list_splice_init(from, &ret_folios);
list_splice_init(&thp_split_pages, from); list_splice_init(&split_folios, from);
no_subpage_counting = true; no_split_folio_counting = true;
retry = 1; retry = 1;
goto thp_subpage_migration; goto split_folio_migration;
} }
rc = nr_failed + nr_thp_failed; rc = nr_failed + nr_large_failed;
out: out:
/* /*
* Put the permanent failure page back to migration list, they * Put the permanent failure folio back to migration list, they
* will be put back to the right list by the caller. * will be put back to the right list by the caller.
*/ */
list_splice(&ret_pages, from); list_splice(&ret_folios, from);
/* /*
* Return 0 in case all subpages of fail-to-migrate THPs are * Return 0 in case all split folios of fail-to-migrate large folios
* migrated successfully. * are migrated successfully.
*/ */
if (list_empty(from)) if (list_empty(from))
rc = 0; rc = 0;