mm/gup: add FOLL_LONGTERM capability to GUP fast
DAX pages were previously unprotected from longterm pins when users called get_user_pages_fast(). Use the new FOLL_LONGTERM flag to check for DEVMAP pages and fall back to regular GUP processing if a DEVMAP page is encountered. [ira.weiny@intel.com: v3] Link: http://lkml.kernel.org/r/20190328084422.29911-5-ira.weiny@intel.com Link: http://lkml.kernel.org/r/20190328084422.29911-5-ira.weiny@intel.com Link: http://lkml.kernel.org/r/20190317183438.2057-5-ira.weiny@intel.com Signed-off-by: Ira Weiny <ira.weiny@intel.com> Reviewed-by: Andrew Morton <akpm@linux-foundation.org> Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Dan Williams <dan.j.williams@intel.com> Cc: "David S. Miller" <davem@davemloft.net> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: James Hogan <jhogan@kernel.org> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: John Hubbard <jhubbard@nvidia.com> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Michal Hocko <mhocko@kernel.org> Cc: Paul Mackerras <paulus@samba.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Ralf Baechle <ralf@linux-mips.org> Cc: Rich Felker <dalias@libc.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Yoshinori Sato <ysato@users.sourceforge.jp> Cc: Mike Marshall <hubcap@omnibond.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
73b0140bf0
commit
7af75561e1
40
mm/gup.c
40
mm/gup.c
|
@ -1637,6 +1637,9 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
|
|||
goto pte_unmap;
|
||||
|
||||
if (pte_devmap(pte)) {
|
||||
if (unlikely(flags & FOLL_LONGTERM))
|
||||
goto pte_unmap;
|
||||
|
||||
pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
|
||||
if (unlikely(!pgmap)) {
|
||||
undo_dev_pagemap(nr, nr_start, pages);
|
||||
|
@ -1776,8 +1779,11 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
|
|||
if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
|
||||
return 0;
|
||||
|
||||
if (pmd_devmap(orig))
|
||||
if (pmd_devmap(orig)) {
|
||||
if (unlikely(flags & FOLL_LONGTERM))
|
||||
return 0;
|
||||
return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);
|
||||
}
|
||||
|
||||
refs = 0;
|
||||
page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
|
||||
|
@ -1814,8 +1820,11 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
|
|||
if (!pud_access_permitted(orig, flags & FOLL_WRITE))
|
||||
return 0;
|
||||
|
||||
if (pud_devmap(orig))
|
||||
if (pud_devmap(orig)) {
|
||||
if (unlikely(flags & FOLL_LONGTERM))
|
||||
return 0;
|
||||
return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);
|
||||
}
|
||||
|
||||
refs = 0;
|
||||
page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
|
||||
|
@ -2058,6 +2067,29 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
|
|||
return nr;
|
||||
}
|
||||
|
||||
static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
|
||||
unsigned int gup_flags, struct page **pages)
|
||||
{
|
||||
int ret;
|
||||
|
||||
/*
|
||||
* FIXME: FOLL_LONGTERM does not work with
|
||||
* get_user_pages_unlocked() (see comments in that function)
|
||||
*/
|
||||
if (gup_flags & FOLL_LONGTERM) {
|
||||
down_read(¤t->mm->mmap_sem);
|
||||
ret = __gup_longterm_locked(current, current->mm,
|
||||
start, nr_pages,
|
||||
pages, NULL, gup_flags);
|
||||
up_read(¤t->mm->mmap_sem);
|
||||
} else {
|
||||
ret = get_user_pages_unlocked(start, nr_pages,
|
||||
pages, gup_flags);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* get_user_pages_fast() - pin user pages in memory
|
||||
* @start: starting user address
|
||||
|
@ -2103,8 +2135,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages,
|
|||
start += nr << PAGE_SHIFT;
|
||||
pages += nr;
|
||||
|
||||
ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
|
||||
gup_flags);
|
||||
ret = __gup_longterm_unlocked(start, nr_pages - nr,
|
||||
gup_flags, pages);
|
||||
|
||||
/* Have to be a bit careful with return values */
|
||||
if (nr > 0) {
|
||||
|
|
Loading…
Reference in New Issue