2013-04-10 20:48:00 +08:00
|
|
|
/*
|
|
|
|
* arch/arm64/mm/hugetlbpage.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 2013 Linaro Ltd.
|
|
|
|
*
|
|
|
|
* Based on arch/x86/mm/hugetlbpage.c.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <asm/mman.h>
|
|
|
|
#include <asm/tlb.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/pgalloc.h>
|
|
|
|
|
2019-03-06 07:43:58 +08:00
|
|
|
#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
|
|
|
|
bool arch_hugetlb_migration_supported(struct hstate *h)
|
|
|
|
{
|
|
|
|
size_t pagesize = huge_page_size(h);
|
|
|
|
|
|
|
|
switch (pagesize) {
|
|
|
|
#ifdef CONFIG_ARM64_4K_PAGES
|
|
|
|
case PUD_SIZE:
|
|
|
|
#endif
|
|
|
|
case PMD_SIZE:
|
|
|
|
case CONT_PMD_SIZE:
|
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
pr_warn("%s: unrecognized huge page size 0x%lx\n",
|
|
|
|
__func__, pagesize);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2013-04-10 20:48:00 +08:00
|
|
|
int pmd_huge(pmd_t pmd)
|
|
|
|
{
|
2015-07-01 20:08:31 +08:00
|
|
|
return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
|
2013-04-10 20:48:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int pud_huge(pud_t pud)
|
|
|
|
{
|
2014-05-15 22:19:22 +08:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2015-07-01 20:08:31 +08:00
|
|
|
return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
|
2014-05-15 22:19:22 +08:00
|
|
|
#else
|
|
|
|
return 0;
|
|
|
|
#endif
|
2013-04-10 20:48:00 +08:00
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:42 +08:00
|
|
|
/*
|
|
|
|
* Select all bits except the pfn
|
|
|
|
*/
|
|
|
|
static inline pgprot_t pte_pgprot(pte_t pte)
|
|
|
|
{
|
|
|
|
unsigned long pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
static int find_num_contig(struct mm_struct *mm, unsigned long addr,
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
pte_t *ptep, size_t *pgsize)
|
2015-12-18 03:31:26 +08:00
|
|
|
{
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp = pgd_offset(mm, addr);
|
|
|
|
pud_t *pudp;
|
|
|
|
pmd_t *pmdp;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
|
|
|
*pgsize = PAGE_SIZE;
|
2018-02-15 19:14:56 +08:00
|
|
|
pudp = pud_offset(pgdp, addr);
|
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
if ((pte_t *)pmdp == ptep) {
|
2015-12-18 03:31:26 +08:00
|
|
|
*pgsize = PMD_SIZE;
|
|
|
|
return CONT_PMDS;
|
|
|
|
}
|
|
|
|
return CONT_PTES;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:46 +08:00
|
|
|
static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
|
|
|
|
{
|
|
|
|
int contig_ptes = 0;
|
|
|
|
|
|
|
|
*pgsize = size;
|
|
|
|
|
|
|
|
switch (size) {
|
|
|
|
#ifdef CONFIG_ARM64_4K_PAGES
|
|
|
|
case PUD_SIZE:
|
|
|
|
#endif
|
|
|
|
case PMD_SIZE:
|
|
|
|
contig_ptes = 1;
|
|
|
|
break;
|
|
|
|
case CONT_PMD_SIZE:
|
|
|
|
*pgsize = PMD_SIZE;
|
|
|
|
contig_ptes = CONT_PMDS;
|
|
|
|
break;
|
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
*pgsize = PAGE_SIZE;
|
|
|
|
contig_ptes = CONT_PTES;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return contig_ptes;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
/*
|
|
|
|
* Changing some bits of contiguous entries requires us to follow a
|
|
|
|
* Break-Before-Make approach, breaking the whole contiguous set
|
|
|
|
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
|
|
|
|
* "Misprogramming of the Contiguous bit", page D4-1762.
|
|
|
|
*
|
|
|
|
* This helper performs the break step.
|
|
|
|
*/
|
|
|
|
static pte_t get_clear_flush(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
|
|
|
pte_t orig_pte = huge_ptep_get(ptep);
|
|
|
|
bool valid = pte_valid(orig_pte);
|
|
|
|
unsigned long i, saddr = addr;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
|
|
|
|
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If HW_AFDBM is enabled, then the HW could turn on
|
2018-09-21 23:34:04 +08:00
|
|
|
* the dirty or accessed bit for any page in the set,
|
|
|
|
* so check them all.
|
2017-08-22 18:42:44 +08:00
|
|
|
*/
|
|
|
|
if (pte_dirty(pte))
|
|
|
|
orig_pte = pte_mkdirty(orig_pte);
|
2018-09-21 23:34:04 +08:00
|
|
|
|
|
|
|
if (pte_young(pte))
|
|
|
|
orig_pte = pte_mkyoung(orig_pte);
|
2017-08-22 18:42:44 +08:00
|
|
|
}
|
|
|
|
|
mm: do not initialize TLB stack vma's with vma_init()
Commit 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and
data segments") tried to initialize various left-over ad-hoc vma's
"properly", but actually made things worse for the temporary vma's used
for TLB flushing.
vma_init() doesn't actually initialize all of the vma, just a few
fields, so doing something like
- struct vm_area_struct vma = { .vm_mm = tlb->mm, };
+ struct vm_area_struct vma;
+
+ vma_init(&vma, tlb->mm);
was actually very bad: instead of having a nicely initialized vma with
every field but "vm_mm" zeroed, you'd have an entirely uninitialized vma
with only a couple of fields initialized. And they weren't even fields
that the code in question mostly cared about.
The flush_tlb_range() function takes a "struct vma" rather than a
"struct mm_struct", because a few architectures actually care about what
kind of range it is - being able to only do an ITLB flush if it's a
range that doesn't have data accesses enabled, for example. And all the
normal users already have the vma for doing the range invalidation.
But a few people want to call flush_tlb_range() with a range they just
made up, so they also end up using a made-up vma. x86 just has a
special "flush_tlb_mm_range()" function for this, but other
architectures (arm and ia64) do the "use fake vma" thing instead, and
thus got caught up in the vma_init() changes.
At the same time, the TLB flushing code really doesn't care about most
other fields in the vma, so vma_init() is just unnecessary and
pointless.
This fixes things by having an explicit "this is just an initializer for
the TLB flush" initializer macro, which is used by the arm/arm64/ia64
people who mis-use this interface with just a dummy vma.
Fixes: 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and data segments")
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2018-08-02 04:43:38 +08:00
|
|
|
if (valid) {
|
|
|
|
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
|
2017-08-22 18:42:44 +08:00
|
|
|
flush_tlb_range(&vma, saddr, addr);
|
mm: do not initialize TLB stack vma's with vma_init()
Commit 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and
data segments") tried to initialize various left-over ad-hoc vma's
"properly", but actually made things worse for the temporary vma's used
for TLB flushing.
vma_init() doesn't actually initialize all of the vma, just a few
fields, so doing something like
- struct vm_area_struct vma = { .vm_mm = tlb->mm, };
+ struct vm_area_struct vma;
+
+ vma_init(&vma, tlb->mm);
was actually very bad: instead of having a nicely initialized vma with
every field but "vm_mm" zeroed, you'd have an entirely uninitialized vma
with only a couple of fields initialized. And they weren't even fields
that the code in question mostly cared about.
The flush_tlb_range() function takes a "struct vma" rather than a
"struct mm_struct", because a few architectures actually care about what
kind of range it is - being able to only do an ITLB flush if it's a
range that doesn't have data accesses enabled, for example. And all the
normal users already have the vma for doing the range invalidation.
But a few people want to call flush_tlb_range() with a range they just
made up, so they also end up using a made-up vma. x86 just has a
special "flush_tlb_mm_range()" function for this, but other
architectures (arm and ia64) do the "use fake vma" thing instead, and
thus got caught up in the vma_init() changes.
At the same time, the TLB flushing code really doesn't care about most
other fields in the vma, so vma_init() is just unnecessary and
pointless.
This fixes things by having an explicit "this is just an initializer for
the TLB flush" initializer macro, which is used by the arm/arm64/ia64
people who mis-use this interface with just a dummy vma.
Fixes: 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and data segments")
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2018-08-02 04:43:38 +08:00
|
|
|
}
|
2017-08-22 18:42:44 +08:00
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Changing some bits of contiguous entries requires us to follow a
|
|
|
|
* Break-Before-Make approach, breaking the whole contiguous set
|
|
|
|
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
|
|
|
|
* "Misprogramming of the Contiguous bit", page D4-1762.
|
|
|
|
*
|
|
|
|
* This helper performs the break step for use cases where the
|
|
|
|
* original pte is not needed.
|
|
|
|
*/
|
|
|
|
static void clear_flush(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
mm: do not initialize TLB stack vma's with vma_init()
Commit 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and
data segments") tried to initialize various left-over ad-hoc vma's
"properly", but actually made things worse for the temporary vma's used
for TLB flushing.
vma_init() doesn't actually initialize all of the vma, just a few
fields, so doing something like
- struct vm_area_struct vma = { .vm_mm = tlb->mm, };
+ struct vm_area_struct vma;
+
+ vma_init(&vma, tlb->mm);
was actually very bad: instead of having a nicely initialized vma with
every field but "vm_mm" zeroed, you'd have an entirely uninitialized vma
with only a couple of fields initialized. And they weren't even fields
that the code in question mostly cared about.
The flush_tlb_range() function takes a "struct vma" rather than a
"struct mm_struct", because a few architectures actually care about what
kind of range it is - being able to only do an ITLB flush if it's a
range that doesn't have data accesses enabled, for example. And all the
normal users already have the vma for doing the range invalidation.
But a few people want to call flush_tlb_range() with a range they just
made up, so they also end up using a made-up vma. x86 just has a
special "flush_tlb_mm_range()" function for this, but other
architectures (arm and ia64) do the "use fake vma" thing instead, and
thus got caught up in the vma_init() changes.
At the same time, the TLB flushing code really doesn't care about most
other fields in the vma, so vma_init() is just unnecessary and
pointless.
This fixes things by having an explicit "this is just an initializer for
the TLB flush" initializer macro, which is used by the arm/arm64/ia64
people who mis-use this interface with just a dummy vma.
Fixes: 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and data segments")
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2018-08-02 04:43:38 +08:00
|
|
|
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
|
2017-08-22 18:42:44 +08:00
|
|
|
unsigned long i, saddr = addr;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
|
|
|
pte_clear(mm, addr, ptep);
|
|
|
|
|
|
|
|
flush_tlb_range(&vma, saddr, addr);
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, pte_t pte)
|
|
|
|
{
|
|
|
|
size_t pgsize;
|
|
|
|
int i;
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
int ncontig;
|
2017-08-22 18:42:43 +08:00
|
|
|
unsigned long pfn, dpfn;
|
2015-12-18 03:31:26 +08:00
|
|
|
pgprot_t hugeprot;
|
|
|
|
|
2017-08-22 18:42:41 +08:00
|
|
|
/*
|
|
|
|
* Code needs to be expanded to handle huge swap and migration
|
|
|
|
* entries. Needed for HUGETLB and MEMORY_FAILURE.
|
|
|
|
*/
|
|
|
|
WARN_ON(!pte_present(pte));
|
|
|
|
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
if (!pte_cont(pte)) {
|
2015-12-18 03:31:26 +08:00
|
|
|
set_pte_at(mm, addr, ptep, pte);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2015-12-18 03:31:26 +08:00
|
|
|
pfn = pte_pfn(pte);
|
2017-08-22 18:42:43 +08:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
2017-08-22 18:42:42 +08:00
|
|
|
hugeprot = pte_pgprot(pte);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
clear_flush(mm, addr, ptep, pgsize, ncontig);
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
2015-12-18 03:31:26 +08:00
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:47 +08:00
|
|
|
void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, pte_t pte, unsigned long sz)
|
|
|
|
{
|
|
|
|
int i, ncontig;
|
|
|
|
size_t pgsize;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(sz, &pgsize);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++)
|
|
|
|
set_pte(ptep, pte);
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
pte_t *huge_pte_alloc(struct mm_struct *mm,
|
|
|
|
unsigned long addr, unsigned long sz)
|
|
|
|
{
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp;
|
|
|
|
pud_t *pudp;
|
|
|
|
pmd_t *pmdp;
|
|
|
|
pte_t *ptep = NULL;
|
|
|
|
|
|
|
|
pgdp = pgd_offset(mm, addr);
|
|
|
|
pudp = pud_alloc(mm, pgdp, addr);
|
|
|
|
if (!pudp)
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (sz == PUD_SIZE) {
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = (pte_t *)pudp;
|
2015-12-18 03:31:26 +08:00
|
|
|
} else if (sz == (PAGE_SIZE * CONT_PTES)) {
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_alloc(mm, pudp, addr);
|
2015-12-18 03:31:26 +08:00
|
|
|
|
|
|
|
WARN_ON(addr & (sz - 1));
|
|
|
|
/*
|
|
|
|
* Note that if this code were ever ported to the
|
|
|
|
* 32-bit arm platform then it will cause trouble in
|
|
|
|
* the case where CONFIG_HIGHPTE is set, since there
|
|
|
|
* will be no pte_unmap() to correspond with this
|
|
|
|
* pte_alloc_map().
|
|
|
|
*/
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = pte_alloc_map(mm, pmdp, addr);
|
2015-12-18 03:31:26 +08:00
|
|
|
} else if (sz == PMD_SIZE) {
|
|
|
|
if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
|
2018-02-15 19:14:56 +08:00
|
|
|
pud_none(READ_ONCE(*pudp)))
|
|
|
|
ptep = huge_pmd_share(mm, addr, pudp);
|
2015-12-18 03:31:26 +08:00
|
|
|
else
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
|
2015-12-18 03:31:26 +08:00
|
|
|
} else if (sz == (PMD_SIZE * CONT_PMDS)) {
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_alloc(mm, pudp, addr);
|
2015-12-18 03:31:26 +08:00
|
|
|
WARN_ON(addr & (sz - 1));
|
2018-02-15 19:14:56 +08:00
|
|
|
return (pte_t *)pmdp;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
return ptep;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2017-07-07 06:39:42 +08:00
|
|
|
pte_t *huge_pte_offset(struct mm_struct *mm,
|
|
|
|
unsigned long addr, unsigned long sz)
|
2015-12-18 03:31:26 +08:00
|
|
|
{
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp;
|
|
|
|
pud_t *pudp, pud;
|
|
|
|
pmd_t *pmdp, pmd;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pgdp = pgd_offset(mm, addr);
|
|
|
|
if (!pgd_present(READ_ONCE(*pgdp)))
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
2017-06-09 01:25:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pudp = pud_offset(pgdp, addr);
|
|
|
|
pud = READ_ONCE(*pudp);
|
|
|
|
if (sz != PUD_SIZE && pud_none(pud))
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
2017-08-22 18:42:45 +08:00
|
|
|
/* hugepage or swap? */
|
2018-02-15 19:14:56 +08:00
|
|
|
if (pud_huge(pud) || !pud_present(pud))
|
|
|
|
return (pte_t *)pudp;
|
2017-06-09 01:25:26 +08:00
|
|
|
/* table; check the next level */
|
|
|
|
|
2017-08-22 18:42:45 +08:00
|
|
|
if (sz == CONT_PMD_SIZE)
|
|
|
|
addr &= CONT_PMD_MASK;
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
pmd = READ_ONCE(*pmdp);
|
2017-08-22 18:42:45 +08:00
|
|
|
if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
|
2018-02-15 19:14:56 +08:00
|
|
|
pmd_none(pmd))
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
2018-02-15 19:14:56 +08:00
|
|
|
if (pmd_huge(pmd) || !pmd_present(pmd))
|
|
|
|
return (pte_t *)pmdp;
|
2017-06-09 01:25:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (sz == CONT_PTE_SIZE)
|
|
|
|
return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));
|
2017-08-22 18:42:45 +08:00
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
|
|
|
|
struct page *page, int writable)
|
|
|
|
{
|
|
|
|
size_t pagesize = huge_page_size(hstate_vma(vma));
|
|
|
|
|
|
|
|
if (pagesize == CONT_PTE_SIZE) {
|
|
|
|
entry = pte_mkcont(entry);
|
|
|
|
} else if (pagesize == CONT_PMD_SIZE) {
|
|
|
|
entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
|
|
|
|
} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
|
|
|
|
pr_warn("%s: unrecognized huge page size 0x%lx\n",
|
|
|
|
__func__, pagesize);
|
|
|
|
}
|
|
|
|
return entry;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:46 +08:00
|
|
|
void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, unsigned long sz)
|
|
|
|
{
|
|
|
|
int i, ncontig;
|
|
|
|
size_t pgsize;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(sz, &pgsize);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
|
|
|
pte_clear(mm, addr, ptep);
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 18:42:44 +08:00
|
|
|
int ncontig;
|
2017-08-22 18:42:43 +08:00
|
|
|
size_t pgsize;
|
|
|
|
pte_t orig_pte = huge_ptep_get(ptep);
|
|
|
|
|
|
|
|
if (!pte_cont(orig_pte))
|
2015-12-18 03:31:26 +08:00
|
|
|
return ptep_get_and_clear(mm, addr, ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
return get_clear_flush(mm, addr, ptep, pgsize, ncontig);
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2018-09-21 23:34:05 +08:00
|
|
|
/*
|
|
|
|
* huge_ptep_set_access_flags will update access flags (dirty, accesssed)
|
|
|
|
* and write permission.
|
|
|
|
*
|
|
|
|
* For a contiguous huge pte range we need to check whether or not write
|
|
|
|
* permission has to change only on the first pte in the set. Then for
|
|
|
|
* all the contiguous ptes we need to check whether or not there is a
|
|
|
|
* discrepancy between dirty or young.
|
|
|
|
*/
|
|
|
|
static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (pte_write(pte) != pte_write(huge_ptep_get(ptep)))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++) {
|
|
|
|
pte_t orig_pte = huge_ptep_get(ptep + i);
|
|
|
|
|
|
|
|
if (pte_dirty(pte) != pte_dirty(orig_pte))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
if (pte_young(pte) != pte_young(orig_pte))
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
int huge_ptep_set_access_flags(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep,
|
|
|
|
pte_t pte, int dirty)
|
|
|
|
{
|
2018-09-21 23:34:05 +08:00
|
|
|
int ncontig, i;
|
2017-08-22 18:42:43 +08:00
|
|
|
size_t pgsize = 0;
|
|
|
|
unsigned long pfn = pte_pfn(pte), dpfn;
|
|
|
|
pgprot_t hugeprot;
|
2017-08-22 18:42:44 +08:00
|
|
|
pte_t orig_pte;
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
if (!pte_cont(pte))
|
2015-12-18 03:31:26 +08:00
|
|
|
return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
|
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
|
|
|
|
2018-09-21 23:34:05 +08:00
|
|
|
if (!__cont_access_flags_changed(ptep, pte, ncontig))
|
|
|
|
return 0;
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
|
|
|
|
|
2018-09-21 23:34:04 +08:00
|
|
|
/* Make sure we don't lose the dirty or young state */
|
2017-08-22 18:42:44 +08:00
|
|
|
if (pte_dirty(orig_pte))
|
|
|
|
pte = pte_mkdirty(pte);
|
|
|
|
|
2018-09-21 23:34:04 +08:00
|
|
|
if (pte_young(orig_pte))
|
|
|
|
pte = pte_mkyoung(pte);
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
hugeprot = pte_pgprot(pte);
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
|
|
|
set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2018-09-21 23:34:05 +08:00
|
|
|
return 1;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 18:42:44 +08:00
|
|
|
unsigned long pfn, dpfn;
|
|
|
|
pgprot_t hugeprot;
|
2017-08-22 18:42:43 +08:00
|
|
|
int ncontig, i;
|
|
|
|
size_t pgsize;
|
2017-08-22 18:42:44 +08:00
|
|
|
pte_t pte;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (!pte_cont(READ_ONCE(*ptep))) {
|
2015-12-18 03:31:26 +08:00
|
|
|
ptep_set_wrprotect(mm, addr, ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
return;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2017-08-22 18:42:44 +08:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig);
|
|
|
|
pte = pte_wrprotect(pte);
|
|
|
|
|
|
|
|
hugeprot = pte_pgprot(pte);
|
|
|
|
pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void huge_ptep_clear_flush(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 18:42:43 +08:00
|
|
|
size_t pgsize;
|
2017-08-22 18:42:44 +08:00
|
|
|
int ncontig;
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (!pte_cont(READ_ONCE(*ptep))) {
|
2015-12-18 03:31:26 +08:00
|
|
|
ptep_clear_flush(vma, addr, ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
return;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
|
2017-08-22 18:42:44 +08:00
|
|
|
clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2018-10-23 09:06:57 +08:00
|
|
|
static void __init add_huge_page_size(unsigned long size)
|
|
|
|
{
|
|
|
|
if (size_to_hstate(size))
|
|
|
|
return;
|
|
|
|
|
|
|
|
hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init hugetlbpage_init(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_ARM64_4K_PAGES
|
|
|
|
add_huge_page_size(PUD_SIZE);
|
|
|
|
#endif
|
|
|
|
add_huge_page_size(PMD_SIZE * CONT_PMDS);
|
|
|
|
add_huge_page_size(PMD_SIZE);
|
|
|
|
add_huge_page_size(PAGE_SIZE * CONT_PTES);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
arch_initcall(hugetlbpage_init);
|
|
|
|
|
2013-04-10 20:48:00 +08:00
|
|
|
static __init int setup_hugepagesz(char *opt)
|
|
|
|
{
|
|
|
|
unsigned long ps = memparse(opt, &opt);
|
2015-12-18 03:31:26 +08:00
|
|
|
|
2017-08-22 18:42:49 +08:00
|
|
|
switch (ps) {
|
|
|
|
#ifdef CONFIG_ARM64_4K_PAGES
|
|
|
|
case PUD_SIZE:
|
|
|
|
#endif
|
|
|
|
case PMD_SIZE * CONT_PMDS:
|
|
|
|
case PMD_SIZE:
|
|
|
|
case PAGE_SIZE * CONT_PTES:
|
2018-10-23 09:06:57 +08:00
|
|
|
add_huge_page_size(ps);
|
2017-08-22 18:42:49 +08:00
|
|
|
return 1;
|
2013-04-10 20:48:00 +08:00
|
|
|
}
|
2017-08-22 18:42:49 +08:00
|
|
|
|
|
|
|
hugetlb_bad_size();
|
|
|
|
pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
|
|
|
|
return 0;
|
2013-04-10 20:48:00 +08:00
|
|
|
}
|
|
|
|
__setup("hugepagesz=", setup_hugepagesz);
|