powerpc/mm: THP is only available on hash64 as of now
Only code movement in this patch. No functionality change. Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
This commit is contained in:
Aneesh Kumar K.V
Michael Ellerman
parent
c0a6c719d2
commit
6a1ea36260
|
|
@ -777,18 +777,6 @@ static inline void vmemmap_remove_mapping(unsigned long start,
|
|||
#endif
|
||||
struct page *realmode_pfn_to_page(unsigned long pfn);
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
|
||||
extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
|
||||
extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
|
||||
extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, pmd_t pmd);
|
||||
extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
pmd_t *pmd);
|
||||
extern int has_transparent_hugepage(void);
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
||||
|
||||
static inline pte_t pmd_pte(pmd_t pmd)
|
||||
{
|
||||
return __pte(pmd_val(pmd));
|
||||
|
|
@ -803,7 +791,6 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd)
|
|||
{
|
||||
return (pte_t *)pmd;
|
||||
}
|
||||
|
||||
#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
|
||||
#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
|
||||
#define pmd_young(pmd) pte_young(pmd_pte(pmd))
|
||||
|
|
@ -830,6 +817,16 @@ static inline int pmd_protnone(pmd_t pmd)
|
|||
#define __HAVE_ARCH_PMD_WRITE
|
||||
#define pmd_write(pmd) pte_write(pmd_pte(pmd))
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
|
||||
extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
|
||||
extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
|
||||
extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, pmd_t pmd);
|
||||
extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
pmd_t *pmd);
|
||||
extern int has_transparent_hugepage(void);
|
||||
|
||||
static inline pmd_t pmd_mkhuge(pmd_t pmd)
|
||||
{
|
||||
return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE));
|
||||
|
|
@ -878,5 +875,6 @@ static inline int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
|
|||
*/
|
||||
return true;
|
||||
}
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@
|
|||
|
||||
#include "mmu_decl.h"
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/thp.h>
|
||||
|
||||
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
||||
/*
|
||||
* On hash-based CPUs, the vmemmap is bolted in the hash table.
|
||||
|
|
@ -93,3 +96,358 @@ int hash__map_kernel_page(unsigned long ea, unsigned long pa, unsigned long flag
|
|||
smp_wmb();
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
|
||||
/*
|
||||
* This is called when relaxing access to a hugepage. It's also called in the page
|
||||
* fault path when we don't hit any of the major fault cases, ie, a minor
|
||||
* update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
|
||||
* handled those two for us, we additionally deal with missing execute
|
||||
* permission here on some processors
|
||||
*/
|
||||
int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp, pmd_t entry, int dirty)
|
||||
{
|
||||
int changed;
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
WARN_ON(!pmd_trans_huge(*pmdp));
|
||||
assert_spin_locked(&vma->vm_mm->page_table_lock);
|
||||
#endif
|
||||
changed = !pmd_same(*(pmdp), entry);
|
||||
if (changed) {
|
||||
__ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry));
|
||||
/*
|
||||
* Since we are not supporting SW TLB systems, we don't
|
||||
* have any thing similar to flush_tlb_page_nohash()
|
||||
*/
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long clr,
|
||||
unsigned long set)
|
||||
{
|
||||
__be64 old_be, tmp;
|
||||
unsigned long old;
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
WARN_ON(!pmd_trans_huge(*pmdp));
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
#endif
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: ldarx %0,0,%3\n\
|
||||
and. %1,%0,%6\n\
|
||||
bne- 1b \n\
|
||||
andc %1,%0,%4 \n\
|
||||
or %1,%1,%7\n\
|
||||
stdcx. %1,0,%3 \n\
|
||||
bne- 1b"
|
||||
: "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
|
||||
: "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
|
||||
"r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
|
||||
: "cc" );
|
||||
|
||||
old = be64_to_cpu(old_be);
|
||||
|
||||
trace_hugepage_update(addr, old, clr, set);
|
||||
if (old & H_PAGE_HASHPTE)
|
||||
hpte_do_hugepage_flush(mm, addr, pmdp, old);
|
||||
return old;
|
||||
}
|
||||
|
||||
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
pmd_t pmd;
|
||||
|
||||
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
||||
VM_BUG_ON(pmd_trans_huge(*pmdp));
|
||||
|
||||
pmd = *pmdp;
|
||||
pmd_clear(pmdp);
|
||||
/*
|
||||
* Wait for all pending hash_page to finish. This is needed
|
||||
* in case of subpage collapse. When we collapse normal pages
|
||||
* to hugepage, we first clear the pmd, then invalidate all
|
||||
* the PTE entries. The assumption here is that any low level
|
||||
* page fault will see a none pmd and take the slow path that
|
||||
* will wait on mmap_sem. But we could very well be in a
|
||||
* hash_page with local ptep pointer value. Such a hash page
|
||||
* can result in adding new HPTE entries for normal subpages.
|
||||
* That means we could be modifying the page content as we
|
||||
* copy them to a huge page. So wait for parallel hash_page
|
||||
* to finish before invalidating HPTE entries. We can do this
|
||||
* by sending an IPI to all the cpus and executing a dummy
|
||||
* function there.
|
||||
*/
|
||||
kick_all_cpus_sync();
|
||||
/*
|
||||
* Now invalidate the hpte entries in the range
|
||||
* covered by pmd. This make sure we take a
|
||||
* fault and will find the pmd as none, which will
|
||||
* result in a major fault which takes mmap_sem and
|
||||
* hence wait for collapse to complete. Without this
|
||||
* the __collapse_huge_page_copy can result in copying
|
||||
* the old content.
|
||||
*/
|
||||
flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
|
||||
return pmd;
|
||||
}
|
||||
|
||||
/*
|
||||
* We currently remove entries from the hashtable regardless of whether
|
||||
* the entry was young or dirty.
|
||||
*
|
||||
* We should be more intelligent about this but for the moment we override
|
||||
* these functions and force a tlb flush unconditionally
|
||||
*/
|
||||
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp);
|
||||
}
|
||||
|
||||
/*
|
||||
* We want to put the pgtable in pmd and use pgtable for tracking
|
||||
* the base page size hptes
|
||||
*/
|
||||
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pgtable)
|
||||
{
|
||||
pgtable_t *pgtable_slot;
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
/*
|
||||
* we store the pgtable in the second half of PMD
|
||||
*/
|
||||
pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
||||
*pgtable_slot = pgtable;
|
||||
/*
|
||||
* expose the deposited pgtable to other cpus.
|
||||
* before we set the hugepage PTE at pmd level
|
||||
* hash fault code looks at the deposted pgtable
|
||||
* to store hash index values.
|
||||
*/
|
||||
smp_wmb();
|
||||
}
|
||||
|
||||
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
|
||||
{
|
||||
pgtable_t pgtable;
|
||||
pgtable_t *pgtable_slot;
|
||||
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
||||
pgtable = *pgtable_slot;
|
||||
/*
|
||||
* Once we withdraw, mark the entry NULL.
|
||||
*/
|
||||
*pgtable_slot = NULL;
|
||||
/*
|
||||
* We store HPTE information in the deposited PTE fragment.
|
||||
* zero out the content on withdraw.
|
||||
*/
|
||||
memset(pgtable, 0, PTE_FRAG_SIZE);
|
||||
return pgtable;
|
||||
}
|
||||
|
||||
void pmdp_huge_split_prepare(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
||||
VM_BUG_ON(REGION_ID(address) != USER_REGION_ID);
|
||||
|
||||
/*
|
||||
* We can't mark the pmd none here, because that will cause a race
|
||||
* against exit_mmap. We need to continue mark pmd TRANS HUGE, while
|
||||
* we spilt, but at the same time we wan't rest of the ppc64 code
|
||||
* not to insert hash pte on this, because we will be modifying
|
||||
* the deposited pgtable in the caller of this function. Hence
|
||||
* clear the _PAGE_USER so that we move the fault handling to
|
||||
* higher level function and that will serialize against ptl.
|
||||
* We need to flush existing hash pte entries here even though,
|
||||
* the translation is still valid, because we will withdraw
|
||||
* pgtable_t after this.
|
||||
*/
|
||||
pmd_hugepage_update(vma->vm_mm, address, pmdp, 0, _PAGE_PRIVILEGED);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* set a new huge pmd. We should not be called for updating
|
||||
* an existing pmd entry. That should go via pmd_hugepage_update.
|
||||
*/
|
||||
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, pmd_t pmd)
|
||||
{
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp)));
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
WARN_ON(!pmd_trans_huge(pmd));
|
||||
#endif
|
||||
trace_hugepage_set_pmd(addr, pmd_val(pmd));
|
||||
return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
|
||||
}
|
||||
|
||||
/*
|
||||
* We use this to invalidate a pmdp entry before switching from a
|
||||
* hugepte to regular pmd entry.
|
||||
*/
|
||||
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
|
||||
|
||||
/*
|
||||
* This ensures that generic code that rely on IRQ disabling
|
||||
* to prevent a parallel THP split work as expected.
|
||||
*/
|
||||
kick_all_cpus_sync();
|
||||
}
|
||||
|
||||
/*
|
||||
* A linux hugepage PMD was changed and the corresponding hash table entries
|
||||
* neesd to be flushed.
|
||||
*/
|
||||
void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long old_pmd)
|
||||
{
|
||||
int ssize;
|
||||
unsigned int psize;
|
||||
unsigned long vsid;
|
||||
unsigned long flags = 0;
|
||||
const struct cpumask *tmp;
|
||||
|
||||
/* get the base page size,vsid and segment size */
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
psize = get_slice_psize(mm, addr);
|
||||
BUG_ON(psize == MMU_PAGE_16M);
|
||||
#endif
|
||||
if (old_pmd & H_PAGE_COMBO)
|
||||
psize = MMU_PAGE_4K;
|
||||
else
|
||||
psize = MMU_PAGE_64K;
|
||||
|
||||
if (!is_kernel_addr(addr)) {
|
||||
ssize = user_segment_size(addr);
|
||||
vsid = get_vsid(mm->context.id, addr, ssize);
|
||||
WARN_ON(vsid == 0);
|
||||
} else {
|
||||
vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
|
||||
ssize = mmu_kernel_ssize;
|
||||
}
|
||||
|
||||
tmp = cpumask_of(smp_processor_id());
|
||||
if (cpumask_equal(mm_cpumask(mm), tmp))
|
||||
flags |= HPTE_LOCAL_UPDATE;
|
||||
|
||||
return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
|
||||
}
|
||||
|
||||
static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
|
||||
{
|
||||
return __pmd(pmd_val(pmd) | pgprot_val(pgprot));
|
||||
}
|
||||
|
||||
pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot)
|
||||
{
|
||||
unsigned long pmdv;
|
||||
|
||||
pmdv = (pfn << PAGE_SHIFT) & PTE_RPN_MASK;
|
||||
return pmd_set_protbits(__pmd(pmdv), pgprot);
|
||||
}
|
||||
|
||||
pmd_t mk_pmd(struct page *page, pgprot_t pgprot)
|
||||
{
|
||||
return pfn_pmd(page_to_pfn(page), pgprot);
|
||||
}
|
||||
|
||||
pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
||||
{
|
||||
unsigned long pmdv;
|
||||
|
||||
pmdv = pmd_val(pmd);
|
||||
pmdv &= _HPAGE_CHG_MASK;
|
||||
return pmd_set_protbits(__pmd(pmdv), newprot);
|
||||
}
|
||||
|
||||
/*
|
||||
* This is called at the end of handling a user page fault, when the
|
||||
* fault has been handled by updating a HUGE PMD entry in the linux page tables.
|
||||
* We use it to preload an HPTE into the hash table corresponding to
|
||||
* the updated linux HUGE PMD entry.
|
||||
*/
|
||||
void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
pmd_t *pmd)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp)
|
||||
{
|
||||
pmd_t old_pmd;
|
||||
pgtable_t pgtable;
|
||||
unsigned long old;
|
||||
pgtable_t *pgtable_slot;
|
||||
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
|
||||
old_pmd = __pmd(old);
|
||||
/*
|
||||
* We have pmd == none and we are holding page_table_lock.
|
||||
* So we can safely go and clear the pgtable hash
|
||||
* index info.
|
||||
*/
|
||||
pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
||||
pgtable = *pgtable_slot;
|
||||
/*
|
||||
* Let's zero out old valid and hash index details
|
||||
* hash fault look at them.
|
||||
*/
|
||||
memset(pgtable, 0, PTE_FRAG_SIZE);
|
||||
/*
|
||||
* Serialize against find_linux_pte_or_hugepte which does lock-less
|
||||
* lookup in page tables with local interrupts disabled. For huge pages
|
||||
* it casts pmd_t to pte_t. Since format of pte_t is different from
|
||||
* pmd_t we want to prevent transit from pmd pointing to page table
|
||||
* to pmd pointing to huge page (and back) while interrupts are disabled.
|
||||
* We clear pmd to possibly replace it with page table pointer in
|
||||
* different code paths. So make sure we wait for the parallel
|
||||
* find_linux_pte_or_hugepage to finish.
|
||||
*/
|
||||
kick_all_cpus_sync();
|
||||
return old_pmd;
|
||||
}
|
||||
|
||||
int has_transparent_hugepage(void)
|
||||
{
|
||||
|
||||
if (!mmu_has_feature(MMU_FTR_16M_PAGE))
|
||||
return 0;
|
||||
/*
|
||||
* We support THP only if PMD_SIZE is 16MB.
|
||||
*/
|
||||
if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
|
||||
return 0;
|
||||
/*
|
||||
* We need to make sure that we support 16MB hugepage in a segement
|
||||
* with base page size 64K or 4K. We only enable THP with a PAGE_SIZE
|
||||
* of 64K.
|
||||
*/
|
||||
/*
|
||||
* If we have 64K HPTE, we will be using that by default
|
||||
*/
|
||||
if (mmu_psize_defs[MMU_PAGE_64K].shift &&
|
||||
(mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
|
||||
return 0;
|
||||
/*
|
||||
* Ok we only have 4K HPTE
|
||||
*/
|
||||
if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
|
|
|||
|
|
@ -55,9 +55,6 @@
|
|||
|
||||
#include "mmu_decl.h"
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/thp.h>
|
||||
|
||||
#ifdef CONFIG_PPC_STD_MMU_64
|
||||
#if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
|
||||
#error TASK_SIZE_USER64 exceeds user VSID range
|
||||
|
|
@ -435,359 +432,3 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
|
|||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
|
||||
/*
|
||||
* This is called when relaxing access to a hugepage. It's also called in the page
|
||||
* fault path when we don't hit any of the major fault cases, ie, a minor
|
||||
* update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
|
||||
* handled those two for us, we additionally deal with missing execute
|
||||
* permission here on some processors
|
||||
*/
|
||||
int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp, pmd_t entry, int dirty)
|
||||
{
|
||||
int changed;
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
WARN_ON(!pmd_trans_huge(*pmdp));
|
||||
assert_spin_locked(&vma->vm_mm->page_table_lock);
|
||||
#endif
|
||||
changed = !pmd_same(*(pmdp), entry);
|
||||
if (changed) {
|
||||
__ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry));
|
||||
/*
|
||||
* Since we are not supporting SW TLB systems, we don't
|
||||
* have any thing similar to flush_tlb_page_nohash()
|
||||
*/
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long clr,
|
||||
unsigned long set)
|
||||
{
|
||||
|
||||
__be64 old_be, tmp;
|
||||
unsigned long old;
|
||||
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
WARN_ON(!pmd_trans_huge(*pmdp));
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
#endif
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: ldarx %0,0,%3\n\
|
||||
and. %1,%0,%6\n\
|
||||
bne- 1b \n\
|
||||
andc %1,%0,%4 \n\
|
||||
or %1,%1,%7\n\
|
||||
stdcx. %1,0,%3 \n\
|
||||
bne- 1b"
|
||||
: "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp)
|
||||
: "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp),
|
||||
"r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set))
|
||||
: "cc" );
|
||||
|
||||
old = be64_to_cpu(old_be);
|
||||
|
||||
trace_hugepage_update(addr, old, clr, set);
|
||||
if (old & H_PAGE_HASHPTE)
|
||||
hpte_do_hugepage_flush(mm, addr, pmdp, old);
|
||||
return old;
|
||||
}
|
||||
|
||||
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
pmd_t pmd;
|
||||
|
||||
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
||||
VM_BUG_ON(pmd_trans_huge(*pmdp));
|
||||
|
||||
pmd = *pmdp;
|
||||
pmd_clear(pmdp);
|
||||
/*
|
||||
* Wait for all pending hash_page to finish. This is needed
|
||||
* in case of subpage collapse. When we collapse normal pages
|
||||
* to hugepage, we first clear the pmd, then invalidate all
|
||||
* the PTE entries. The assumption here is that any low level
|
||||
* page fault will see a none pmd and take the slow path that
|
||||
* will wait on mmap_sem. But we could very well be in a
|
||||
* hash_page with local ptep pointer value. Such a hash page
|
||||
* can result in adding new HPTE entries for normal subpages.
|
||||
* That means we could be modifying the page content as we
|
||||
* copy them to a huge page. So wait for parallel hash_page
|
||||
* to finish before invalidating HPTE entries. We can do this
|
||||
* by sending an IPI to all the cpus and executing a dummy
|
||||
* function there.
|
||||
*/
|
||||
kick_all_cpus_sync();
|
||||
/*
|
||||
* Now invalidate the hpte entries in the range
|
||||
* covered by pmd. This make sure we take a
|
||||
* fault and will find the pmd as none, which will
|
||||
* result in a major fault which takes mmap_sem and
|
||||
* hence wait for collapse to complete. Without this
|
||||
* the __collapse_huge_page_copy can result in copying
|
||||
* the old content.
|
||||
*/
|
||||
flush_tlb_pmd_range(vma->vm_mm, &pmd, address);
|
||||
return pmd;
|
||||
}
|
||||
|
||||
/*
|
||||
* We currently remove entries from the hashtable regardless of whether
|
||||
* the entry was young or dirty.
|
||||
*
|
||||
* We should be more intelligent about this but for the moment we override
|
||||
* these functions and force a tlb flush unconditionally
|
||||
*/
|
||||
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp);
|
||||
}
|
||||
|
||||
/*
|
||||
* We want to put the pgtable in pmd and use pgtable for tracking
|
||||
* the base page size hptes
|
||||
*/
|
||||
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
|
||||
pgtable_t pgtable)
|
||||
{
|
||||
pgtable_t *pgtable_slot;
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
/*
|
||||
* we store the pgtable in the second half of PMD
|
||||
*/
|
||||
pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
||||
*pgtable_slot = pgtable;
|
||||
/*
|
||||
* expose the deposited pgtable to other cpus.
|
||||
* before we set the hugepage PTE at pmd level
|
||||
* hash fault code looks at the deposted pgtable
|
||||
* to store hash index values.
|
||||
*/
|
||||
smp_wmb();
|
||||
}
|
||||
|
||||
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
|
||||
{
|
||||
pgtable_t pgtable;
|
||||
pgtable_t *pgtable_slot;
|
||||
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
||||
pgtable = *pgtable_slot;
|
||||
/*
|
||||
* Once we withdraw, mark the entry NULL.
|
||||
*/
|
||||
*pgtable_slot = NULL;
|
||||
/*
|
||||
* We store HPTE information in the deposited PTE fragment.
|
||||
* zero out the content on withdraw.
|
||||
*/
|
||||
memset(pgtable, 0, PTE_FRAG_SIZE);
|
||||
return pgtable;
|
||||
}
|
||||
|
||||
void pmdp_huge_split_prepare(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
|
||||
VM_BUG_ON(REGION_ID(address) != USER_REGION_ID);
|
||||
|
||||
/*
|
||||
* We can't mark the pmd none here, because that will cause a race
|
||||
* against exit_mmap. We need to continue mark pmd TRANS HUGE, while
|
||||
* we spilt, but at the same time we wan't rest of the ppc64 code
|
||||
* not to insert hash pte on this, because we will be modifying
|
||||
* the deposited pgtable in the caller of this function. Hence
|
||||
* clear the _PAGE_USER so that we move the fault handling to
|
||||
* higher level function and that will serialize against ptl.
|
||||
* We need to flush existing hash pte entries here even though,
|
||||
* the translation is still valid, because we will withdraw
|
||||
* pgtable_t after this.
|
||||
*/
|
||||
pmd_hugepage_update(vma->vm_mm, address, pmdp, 0, _PAGE_PRIVILEGED);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* set a new huge pmd. We should not be called for updating
|
||||
* an existing pmd entry. That should go via pmd_hugepage_update.
|
||||
*/
|
||||
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, pmd_t pmd)
|
||||
{
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp)));
|
||||
assert_spin_locked(&mm->page_table_lock);
|
||||
WARN_ON(!pmd_trans_huge(pmd));
|
||||
#endif
|
||||
trace_hugepage_set_pmd(addr, pmd_val(pmd));
|
||||
return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
|
||||
}
|
||||
|
||||
/*
|
||||
* We use this to invalidate a pmdp entry before switching from a
|
||||
* hugepte to regular pmd entry.
|
||||
*/
|
||||
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
|
||||
|
||||
/*
|
||||
* This ensures that generic code that rely on IRQ disabling
|
||||
* to prevent a parallel THP split work as expected.
|
||||
*/
|
||||
kick_all_cpus_sync();
|
||||
}
|
||||
|
||||
/*
|
||||
* A linux hugepage PMD was changed and the corresponding hash table entries
|
||||
* neesd to be flushed.
|
||||
*/
|
||||
void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long old_pmd)
|
||||
{
|
||||
int ssize;
|
||||
unsigned int psize;
|
||||
unsigned long vsid;
|
||||
unsigned long flags = 0;
|
||||
const struct cpumask *tmp;
|
||||
|
||||
/* get the base page size,vsid and segment size */
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
psize = get_slice_psize(mm, addr);
|
||||
BUG_ON(psize == MMU_PAGE_16M);
|
||||
#endif
|
||||
if (old_pmd & H_PAGE_COMBO)
|
||||
psize = MMU_PAGE_4K;
|
||||
else
|
||||
psize = MMU_PAGE_64K;
|
||||
|
||||
if (!is_kernel_addr(addr)) {
|
||||
ssize = user_segment_size(addr);
|
||||
vsid = get_vsid(mm->context.id, addr, ssize);
|
||||
WARN_ON(vsid == 0);
|
||||
} else {
|
||||
vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
|
||||
ssize = mmu_kernel_ssize;
|
||||
}
|
||||
|
||||
tmp = cpumask_of(smp_processor_id());
|
||||
if (cpumask_equal(mm_cpumask(mm), tmp))
|
||||
flags |= HPTE_LOCAL_UPDATE;
|
||||
|
||||
return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags);
|
||||
}
|
||||
|
||||
static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
|
||||
{
|
||||
return __pmd(pmd_val(pmd) | pgprot_val(pgprot));
|
||||
}
|
||||
|
||||
pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot)
|
||||
{
|
||||
unsigned long pmdv;
|
||||
|
||||
pmdv = (pfn << PAGE_SHIFT) & PTE_RPN_MASK;
|
||||
return pmd_set_protbits(__pmd(pmdv), pgprot);
|
||||
}
|
||||
|
||||
pmd_t mk_pmd(struct page *page, pgprot_t pgprot)
|
||||
{
|
||||
return pfn_pmd(page_to_pfn(page), pgprot);
|
||||
}
|
||||
|
||||
pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
||||
{
|
||||
unsigned long pmdv;
|
||||
|
||||
pmdv = pmd_val(pmd);
|
||||
pmdv &= _HPAGE_CHG_MASK;
|
||||
return pmd_set_protbits(__pmd(pmdv), newprot);
|
||||
}
|
||||
|
||||
/*
|
||||
* This is called at the end of handling a user page fault, when the
|
||||
* fault has been handled by updating a HUGE PMD entry in the linux page tables.
|
||||
* We use it to preload an HPTE into the hash table corresponding to
|
||||
* the updated linux HUGE PMD entry.
|
||||
*/
|
||||
void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
|
||||
pmd_t *pmd)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp)
|
||||
{
|
||||
pmd_t old_pmd;
|
||||
pgtable_t pgtable;
|
||||
unsigned long old;
|
||||
pgtable_t *pgtable_slot;
|
||||
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
|
||||
old_pmd = __pmd(old);
|
||||
/*
|
||||
* We have pmd == none and we are holding page_table_lock.
|
||||
* So we can safely go and clear the pgtable hash
|
||||
* index info.
|
||||
*/
|
||||
pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD;
|
||||
pgtable = *pgtable_slot;
|
||||
/*
|
||||
* Let's zero out old valid and hash index details
|
||||
* hash fault look at them.
|
||||
*/
|
||||
memset(pgtable, 0, PTE_FRAG_SIZE);
|
||||
/*
|
||||
* Serialize against find_linux_pte_or_hugepte which does lock-less
|
||||
* lookup in page tables with local interrupts disabled. For huge pages
|
||||
* it casts pmd_t to pte_t. Since format of pte_t is different from
|
||||
* pmd_t we want to prevent transit from pmd pointing to page table
|
||||
* to pmd pointing to huge page (and back) while interrupts are disabled.
|
||||
* We clear pmd to possibly replace it with page table pointer in
|
||||
* different code paths. So make sure we wait for the parallel
|
||||
* find_linux_pte_or_hugepage to finish.
|
||||
*/
|
||||
kick_all_cpus_sync();
|
||||
return old_pmd;
|
||||
}
|
||||
|
||||
int has_transparent_hugepage(void)
|
||||
{
|
||||
|
||||
if (!mmu_has_feature(MMU_FTR_16M_PAGE))
|
||||
return 0;
|
||||
/*
|
||||
* We support THP only if PMD_SIZE is 16MB.
|
||||
*/
|
||||
if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT)
|
||||
return 0;
|
||||
/*
|
||||
* We need to make sure that we support 16MB hugepage in a segement
|
||||
* with base page size 64K or 4K. We only enable THP with a PAGE_SIZE
|
||||
* of 64K.
|
||||
*/
|
||||
/*
|
||||
* If we have 64K HPTE, we will be using that by default
|
||||
*/
|
||||
if (mmu_psize_defs[MMU_PAGE_64K].shift &&
|
||||
(mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1))
|
||||
return 0;
|
||||
/*
|
||||
* Ok we only have 4K HPTE
|
||||
*/
|
||||
if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1)
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
|
|
|||
Loading…
Reference in New Issue