UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

x86/pgtable/pae: Use separate kernel PMDs for user page-table 

When PTI is enabled, separate kernel PMDs in the user page-table are
required to map the per-process LDT for user-space.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Pavel Machek <pavel@ucw.cz>
Cc: "H . Peter Anvin" <hpa@zytor.com>
Cc: linux-mm@kvack.org
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: David Laight <David.Laight@aculab.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Eduardo Valentin <eduval@amazon.com>
Cc: Greg KH <gregkh@linuxfoundation.org>
Cc: Will Deacon <will.deacon@arm.com>
Cc: aliguori@amazon.com
Cc: daniel.gruss@iaik.tugraz.at
Cc: hughd@google.com
Cc: keescook@google.com
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Waiman Long <llong@redhat.com>
Cc: "David H . Gutteridge" <dhgutteridge@sympatico.ca>
Cc: joro@8bytes.org
Link: https://lkml.kernel.org/r/1531906876-13451-33-git-send-email-joro@8bytes.org
This commit is contained in:
Joerg Roedel 2018-07-18 11:41:09 +02:00 committed by Thomas Gleixner
parent 4e8537e4a7
commit f59dbe9ca6
1 changed files with 81 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
arch/x86/mm/pgtable.c
View File

@ -182,6 +182,14 @@ static void pgd_dtor(pgd_t *pgd)
*/ */
#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD #define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
/*
* We allocate separate PMDs for the kernel part of the user page-table
* when PTI is enabled. We need them to map the per-process LDT into the
* user-space page-table.
*/
#define PREALLOCATED_USER_PMDS (static_cpu_has(X86_FEATURE_PTI) ? \
KERNEL_PGD_PTRS : 0)
void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
{ {
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
@ -202,14 +210,14 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
/* No need to prepopulate any pagetable entries in non-PAE modes. */ /* No need to prepopulate any pagetable entries in non-PAE modes. */
#define PREALLOCATED_PMDS 0 #define PREALLOCATED_PMDS 0
#define PREALLOCATED_USER_PMDS 0
#endif /* CONFIG_X86_PAE */ #endif /* CONFIG_X86_PAE */
static void free_pmds(struct mm_struct *mm, pmd_t *pmds[]) static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
{ {
int i; int i;
for(i = 0; i < PREALLOCATED_PMDS; i++) for (i = 0; i < count; i++)
if (pmds[i]) { if (pmds[i]) {
pgtable_pmd_page_dtor(virt_to_page(pmds[i])); pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
free_page((unsigned long)pmds[i]); free_page((unsigned long)pmds[i]);
@ -217,7 +225,7 @@ static void free_pmds(struct mm_struct *mm, pmd_t *pmds[])
} }
} }
static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[]) static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
{ {
int i; int i;
bool failed = false; bool failed = false;
@ -226,7 +234,7 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[])
if (mm == &init_mm) if (mm == &init_mm)
gfp &= ~__GFP_ACCOUNT; gfp &= ~__GFP_ACCOUNT;
for(i = 0; i < PREALLOCATED_PMDS; i++) { for (i = 0; i < count; i++) {
pmd_t *pmd = (pmd_t *)__get_free_page(gfp); pmd_t *pmd = (pmd_t *)__get_free_page(gfp);
if (!pmd) if (!pmd)
failed = true; failed = true;
@ -241,7 +249,7 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[])
} }
if (failed) { if (failed) {
free_pmds(mm, pmds); free_pmds(mm, pmds, count);
return -ENOMEM; return -ENOMEM;
} }
@ -254,23 +262,38 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[])
* preallocate which never got a corresponding vma will need to be * preallocate which never got a corresponding vma will need to be
* freed manually. * freed manually.
*/ */
static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp)
{ {
int i; pgd_t pgd = *pgdp;
for(i = 0; i < PREALLOCATED_PMDS; i++) {
pgd_t pgd = pgdp[i];
if (pgd_val(pgd) != 0) { if (pgd_val(pgd) != 0) {
pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd); pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);
pgdp[i] = native_make_pgd(0); *pgdp = native_make_pgd(0);
paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT); paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
pmd_free(mm, pmd); pmd_free(mm, pmd);
mm_dec_nr_pmds(mm); mm_dec_nr_pmds(mm);
} }
} }
static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
{
int i;
for (i = 0; i < PREALLOCATED_PMDS; i++)
mop_up_one_pmd(mm, &pgdp[i]);
#ifdef CONFIG_PAGE_TABLE_ISOLATION
if (!static_cpu_has(X86_FEATURE_PTI))
return;
pgdp = kernel_to_user_pgdp(pgdp);
for (i = 0; i < PREALLOCATED_USER_PMDS; i++)
mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]);
#endif
} }
static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[]) static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
@ -296,6 +319,38 @@ static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
} }
} }
#ifdef CONFIG_PAGE_TABLE_ISOLATION
static void pgd_prepopulate_user_pmd(struct mm_struct *mm,
pgd_t *k_pgd, pmd_t *pmds[])
{
pgd_t *s_pgd = kernel_to_user_pgdp(swapper_pg_dir);
pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd);
p4d_t *u_p4d;
pud_t *u_pud;
int i;
u_p4d = p4d_offset(u_pgd, 0);
u_pud = pud_offset(u_p4d, 0);
s_pgd += KERNEL_PGD_BOUNDARY;
u_pud += KERNEL_PGD_BOUNDARY;
for (i = 0; i < PREALLOCATED_USER_PMDS; i++, u_pud++, s_pgd++) {
pmd_t *pmd = pmds[i];
memcpy(pmd, (pmd_t *)pgd_page_vaddr(*s_pgd),
sizeof(pmd_t) * PTRS_PER_PMD);
pud_populate(mm, u_pud, pmd);
}
}
#else
static void pgd_prepopulate_user_pmd(struct mm_struct *mm,
pgd_t *k_pgd, pmd_t *pmds[])
{
}
#endif
/* /*
* Xen paravirt assumes pgd table should be in one page. 64 bit kernel also * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also
* assumes that pgd should be in one page. * assumes that pgd should be in one page.
@ -376,6 +431,7 @@ static inline void _pgd_free(pgd_t *pgd)
pgd_t *pgd_alloc(struct mm_struct *mm) pgd_t *pgd_alloc(struct mm_struct *mm)
{ {
pgd_t *pgd; pgd_t *pgd;
pmd_t *u_pmds[PREALLOCATED_USER_PMDS];
pmd_t *pmds[PREALLOCATED_PMDS]; pmd_t *pmds[PREALLOCATED_PMDS];
pgd = _pgd_alloc(); pgd = _pgd_alloc();
@ -385,12 +441,15 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
mm->pgd = pgd; mm->pgd = pgd;
if (preallocate_pmds(mm, pmds) != 0) if (preallocate_pmds(mm, pmds, PREALLOCATED_PMDS) != 0)
goto out_free_pgd; goto out_free_pgd;
if (paravirt_pgd_alloc(mm) != 0) if (preallocate_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS) != 0)
goto out_free_pmds; goto out_free_pmds;
if (paravirt_pgd_alloc(mm) != 0)
goto out_free_user_pmds;
/* /*
* Make sure that pre-populating the pmds is atomic with * Make sure that pre-populating the pmds is atomic with
* respect to anything walking the pgd_list, so that they * respect to anything walking the pgd_list, so that they
@ -400,13 +459,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
pgd_ctor(mm, pgd); pgd_ctor(mm, pgd);
pgd_prepopulate_pmd(mm, pgd, pmds); pgd_prepopulate_pmd(mm, pgd, pmds);
pgd_prepopulate_user_pmd(mm, pgd, u_pmds);
spin_unlock(&pgd_lock); spin_unlock(&pgd_lock);
return pgd; return pgd;
out_free_user_pmds:
free_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS);
out_free_pmds: out_free_pmds:
free_pmds(mm, pmds); free_pmds(mm, pmds, PREALLOCATED_PMDS);
out_free_pgd: out_free_pgd:
_pgd_free(pgd); _pgd_free(pgd);
out: out: