hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 mm updates from Ingo Molnar:
 "The main changes in this cycle were:

   - Update and clean up x86 fault handling, by Andy Lutomirski.

   - Drop usage of __flush_tlb_all() in kernel_physical_mapping_init()
     and related fallout, by Dan Williams.

   - CPA cleanups and reorganization by Peter Zijlstra: simplify the
     flow and remove a few warts.

   - Other misc cleanups"

* 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (29 commits)
  x86/mm/dump_pagetables: Use DEFINE_SHOW_ATTRIBUTE()
  x86/mm/cpa: Rename @addrinarray to @numpages
  x86/mm/cpa: Better use CLFLUSHOPT
  x86/mm/cpa: Fold cpa_flush_range() and cpa_flush_array() into a single cpa_flush() function
  x86/mm/cpa: Make cpa_data::numpages invariant
  x86/mm/cpa: Optimize cpa_flush_array() TLB invalidation
  x86/mm/cpa: Simplify the code after making cpa->vaddr invariant
  x86/mm/cpa: Make cpa_data::vaddr invariant
  x86/mm/cpa: Add __cpa_addr() helper
  x86/mm/cpa: Add ARRAY and PAGES_ARRAY selftests
  x86/mm: Drop usage of __flush_tlb_all() in kernel_physical_mapping_init()
  x86/mm: Validate kernel_physical_mapping_init() PTE population
  generic/pgtable: Introduce set_pte_safe()
  generic/pgtable: Introduce {p4d,pgd}_same()
  generic/pgtable: Make {pmd, pud}_same() unconditionally available
  x86/fault: Clean up the page fault oops decoder a bit
  x86/fault: Decode page fault OOPSes better
  x86/vsyscall/64: Use X86_PF constants in the simulated #PF error code
  x86/oops: Show the correct CS value in show_regs()
  x86/fault: Don't try to recover from an implicit supervisor access
  ...
This commit is contained in:
Linus Torvalds 2018-12-26 18:08:18 -08:00
parent d6e867a6ae 6848ac7ca3
commit e57d9f638a
16 changed files with 405 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/x86/entry/vsyscall/vsyscall_64.c
View File

@ -102,7 +102,7 @@ static bool write_ok_or_segv(unsigned long ptr, size_t size)
if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) { if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) {
struct thread_struct *thread = &current->thread; struct thread_struct *thread = &current->thread;
thread->error_code = 6; /* user fault, no page, write */ thread->error_code = X86_PF_USER | X86_PF_WRITE;
thread->cr2 = ptr; thread->cr2 = ptr;
thread->trap_nr = X86_TRAP_PF; thread->trap_nr = X86_TRAP_PF;

8
arch/x86/include/asm/disabled-features.h
View File

@ -16,6 +16,12 @@
# define DISABLE_MPX (1<<(X86_FEATURE_MPX & 31)) # define DISABLE_MPX (1<<(X86_FEATURE_MPX & 31))
#endif #endif
#ifdef CONFIG_X86_SMAP
# define DISABLE_SMAP 0
#else
# define DISABLE_SMAP (1<<(X86_FEATURE_SMAP & 31))
#endif
#ifdef CONFIG_X86_INTEL_UMIP #ifdef CONFIG_X86_INTEL_UMIP
# define DISABLE_UMIP 0 # define DISABLE_UMIP 0
#else #else
@ -68,7 +74,7 @@
#define DISABLED_MASK6 0 #define DISABLED_MASK6 0
#define DISABLED_MASK7 (DISABLE_PTI) #define DISABLED_MASK7 (DISABLE_PTI)
#define DISABLED_MASK8 0 #define DISABLED_MASK8 0
#define DISABLED_MASK9 (DISABLE_MPX) #define DISABLED_MASK9 (DISABLE_MPX|DISABLE_SMAP)
#define DISABLED_MASK10 0 #define DISABLED_MASK10 0
#define DISABLED_MASK11 0 #define DISABLED_MASK11 0
#define DISABLED_MASK12 0 #define DISABLED_MASK12 0

27
arch/x86/include/asm/pgalloc.h
View File

@ -80,6 +80,13 @@ static inline void pmd_populate_kernel(struct mm_struct *mm,
set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
} }
static inline void pmd_populate_kernel_safe(struct mm_struct *mm,
pmd_t *pmd, pte_t *pte)
{
paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
}
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
struct page *pte) struct page *pte)
{ {
@ -132,6 +139,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd))); set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
} }
static inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd)));
}
#endif /* CONFIG_X86_PAE */ #endif /* CONFIG_X86_PAE */
#if CONFIG_PGTABLE_LEVELS > 3 #if CONFIG_PGTABLE_LEVELS > 3
@ -141,6 +154,12 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
} }
static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
{
paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
}
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{ {
gfp_t gfp = GFP_KERNEL_ACCOUNT; gfp_t gfp = GFP_KERNEL_ACCOUNT;
@ -173,6 +192,14 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
} }
static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
{
if (!pgtable_l5_enabled())
return;
paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
}
static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
{ {
gfp_t gfp = GFP_KERNEL_ACCOUNT; gfp_t gfp = GFP_KERNEL_ACCOUNT;

5
arch/x86/kernel/process_64.c
View File

@ -68,7 +68,7 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
unsigned long d0, d1, d2, d3, d6, d7; unsigned long d0, d1, d2, d3, d6, d7;
unsigned int fsindex, gsindex; unsigned int fsindex, gsindex;
unsigned int ds, cs, es; unsigned int ds, es;
show_iret_regs(regs); show_iret_regs(regs);
@ -100,7 +100,6 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
} }
asm("movl %%ds,%0" : "=r" (ds)); asm("movl %%ds,%0" : "=r" (ds));
asm("movl %%cs,%0" : "=r" (cs));
asm("movl %%es,%0" : "=r" (es)); asm("movl %%es,%0" : "=r" (es));
asm("movl %%fs,%0" : "=r" (fsindex)); asm("movl %%fs,%0" : "=r" (fsindex));
asm("movl %%gs,%0" : "=r" (gsindex)); asm("movl %%gs,%0" : "=r" (gsindex));
@ -116,7 +115,7 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n", printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
fs, fsindex, gs, gsindex, shadowgs); fs, fsindex, gs, gsindex, shadowgs);
printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, printk(KERN_DEFAULT "CS: %04lx DS: %04x ES: %04x CR0: %016lx\n", regs->cs, ds,
es, cr0); es, cr0);
printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
cr4); cr4);

58
arch/x86/mm/debug_pagetables.c
View File

@ -10,20 +10,9 @@ static int ptdump_show(struct seq_file *m, void *v)
return 0; return 0;
} }
static int ptdump_open(struct inode *inode, struct file *filp) DEFINE_SHOW_ATTRIBUTE(ptdump);
{
return single_open(filp, ptdump_show, NULL);
}
static const struct file_operations ptdump_fops = { static int ptdump_curknl_show(struct seq_file *m, void *v)
.owner = THIS_MODULE,
.open = ptdump_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static int ptdump_show_curknl(struct seq_file *m, void *v)
{ {
if (current->mm->pgd) { if (current->mm->pgd) {
down_read(&current->mm->mmap_sem); down_read(&current->mm->mmap_sem);
@ -33,23 +22,12 @@ static int ptdump_show_curknl(struct seq_file *m, void *v)
return 0; return 0;
} }
static int ptdump_open_curknl(struct inode *inode, struct file *filp) DEFINE_SHOW_ATTRIBUTE(ptdump_curknl);
{
return single_open(filp, ptdump_show_curknl, NULL);
}
static const struct file_operations ptdump_curknl_fops = {
.owner = THIS_MODULE,
.open = ptdump_open_curknl,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
#ifdef CONFIG_PAGE_TABLE_ISOLATION #ifdef CONFIG_PAGE_TABLE_ISOLATION
static struct dentry *pe_curusr; static struct dentry *pe_curusr;
static int ptdump_show_curusr(struct seq_file *m, void *v) static int ptdump_curusr_show(struct seq_file *m, void *v)
{ {
if (current->mm->pgd) { if (current->mm->pgd) {
down_read(&current->mm->mmap_sem); down_read(&current->mm->mmap_sem);
@ -59,42 +37,20 @@ static int ptdump_show_curusr(struct seq_file *m, void *v)
return 0; return 0;
} }
static int ptdump_open_curusr(struct inode *inode, struct file *filp) DEFINE_SHOW_ATTRIBUTE(ptdump_curusr);
{
return single_open(filp, ptdump_show_curusr, NULL);
}
static const struct file_operations ptdump_curusr_fops = {
.owner = THIS_MODULE,
.open = ptdump_open_curusr,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
#endif #endif
#if defined(CONFIG_EFI) && defined(CONFIG_X86_64) #if defined(CONFIG_EFI) && defined(CONFIG_X86_64)
static struct dentry *pe_efi; static struct dentry *pe_efi;
static int ptdump_show_efi(struct seq_file *m, void *v) static int ptdump_efi_show(struct seq_file *m, void *v)
{ {
if (efi_mm.pgd) if (efi_mm.pgd)
ptdump_walk_pgd_level_debugfs(m, efi_mm.pgd, false); ptdump_walk_pgd_level_debugfs(m, efi_mm.pgd, false);
return 0; return 0;
} }
static int ptdump_open_efi(struct inode *inode, struct file *filp) DEFINE_SHOW_ATTRIBUTE(ptdump_efi);
{
return single_open(filp, ptdump_show_efi, NULL);
}
static const struct file_operations ptdump_efi_fops = {
.owner = THIS_MODULE,
.open = ptdump_open_efi,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
#endif #endif
static struct dentry *dir, *pe_knl, *pe_curknl; static struct dentry *dir, *pe_knl, *pe_curknl;

246
arch/x86/mm/fault.c
View File

@ -27,6 +27,7 @@
#include <asm/vm86.h> /* struct vm86 */ #include <asm/vm86.h> /* struct vm86 */
#include <asm/mmu_context.h> /* vma_pkey() */ #include <asm/mmu_context.h> /* vma_pkey() */
#include <asm/efi.h> /* efi_recover_from_page_fault()*/ #include <asm/efi.h> /* efi_recover_from_page_fault()*/
#include <asm/desc.h> /* store_idt(), ... */
#define CREATE_TRACE_POINTS #define CREATE_TRACE_POINTS
#include <asm/trace/exceptions.h> #include <asm/trace/exceptions.h>
@ -571,10 +572,55 @@ static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
return 0; return 0;
} }
static void static void show_ldttss(const struct desc_ptr *gdt, const char *name, u16 index)
show_fault_oops(struct pt_regs *regs, unsigned long error_code,
unsigned long address)
{ {
u32 offset = (index >> 3) * sizeof(struct desc_struct);
unsigned long addr;
struct ldttss_desc desc;
if (index == 0) {
pr_alert("%s: NULL\n", name);
return;
}
if (offset + sizeof(struct ldttss_desc) >= gdt->size) {
pr_alert("%s: 0x%hx -- out of bounds\n", name, index);
return;
}
if (probe_kernel_read(&desc, (void *)(gdt->address + offset),
sizeof(struct ldttss_desc))) {
pr_alert("%s: 0x%hx -- GDT entry is not readable\n",
name, index);
return;
}
addr = desc.base0 | (desc.base1 << 16) | (desc.base2 << 24);
#ifdef CONFIG_X86_64
addr |= ((u64)desc.base3 << 32);
#endif
pr_alert("%s: 0x%hx -- base=0x%lx limit=0x%x\n",
name, index, addr, (desc.limit0 | (desc.limit1 << 16)));
}
/*
* This helper function transforms the #PF error_code bits into
* "[PROT] [USER]" type of descriptive, almost human-readable error strings:
*/
static void err_str_append(unsigned long error_code, char *buf, unsigned long mask, const char *txt)
{
if (error_code & mask) {
if (buf[0])
strcat(buf, " ");
strcat(buf, txt);
}
}
static void
show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long address)
{
char err_txt[64];
if (!oops_may_print()) if (!oops_may_print())
return; return;
@ -602,6 +648,52 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code,
address < PAGE_SIZE ? "NULL pointer dereference" : "paging request", address < PAGE_SIZE ? "NULL pointer dereference" : "paging request",
(void *)address); (void *)address);
err_txt[0] = 0;
/*
* Note: length of these appended strings including the separation space and the
* zero delimiter must fit into err_txt[].
*/
err_str_append(error_code, err_txt, X86_PF_PROT, "[PROT]" );
err_str_append(error_code, err_txt, X86_PF_WRITE, "[WRITE]");
err_str_append(error_code, err_txt, X86_PF_USER, "[USER]" );
err_str_append(error_code, err_txt, X86_PF_RSVD, "[RSVD]" );
err_str_append(error_code, err_txt, X86_PF_INSTR, "[INSTR]");
err_str_append(error_code, err_txt, X86_PF_PK, "[PK]" );
pr_alert("#PF error: %s\n", error_code ? err_txt : "[normal kernel read fault]");
if (!(error_code & X86_PF_USER) && user_mode(regs)) {
struct desc_ptr idt, gdt;
u16 ldtr, tr;
pr_alert("This was a system access from user code\n");
/*
* This can happen for quite a few reasons. The more obvious
* ones are faults accessing the GDT, or LDT. Perhaps
* surprisingly, if the CPU tries to deliver a benign or
* contributory exception from user code and gets a page fault
* during delivery, the page fault can be delivered as though
* it originated directly from user code. This could happen
* due to wrong permissions on the IDT, GDT, LDT, TSS, or
* kernel or IST stack.
*/
store_idt(&idt);
/* Usable even on Xen PV -- it's just slow. */
native_store_gdt(&gdt);
pr_alert("IDT: 0x%lx (limit=0x%hx) GDT: 0x%lx (limit=0x%hx)\n",
idt.address, idt.size, gdt.address, gdt.size);
store_ldt(ldtr);
show_ldttss(&gdt, "LDTR", ldtr);
store_tr(tr);
show_ldttss(&gdt, "TR", tr);
}
dump_pagetable(address); dump_pagetable(address);
} }
@ -621,16 +713,30 @@ pgtable_bad(struct pt_regs *regs, unsigned long error_code,
tsk->comm, address); tsk->comm, address);
dump_pagetable(address); dump_pagetable(address);
tsk->thread.cr2 = address;
tsk->thread.trap_nr = X86_TRAP_PF;
tsk->thread.error_code = error_code;
if (__die("Bad pagetable", regs, error_code)) if (__die("Bad pagetable", regs, error_code))
sig = 0; sig = 0;
oops_end(flags, regs, sig); oops_end(flags, regs, sig);
} }
static void set_signal_archinfo(unsigned long address,
unsigned long error_code)
{
struct task_struct *tsk = current;
/*
* To avoid leaking information about the kernel page
* table layout, pretend that user-mode accesses to
* kernel addresses are always protection faults.
*/
if (address >= TASK_SIZE_MAX)
error_code |= X86_PF_PROT;
tsk->thread.trap_nr = X86_TRAP_PF;
tsk->thread.error_code = error_code | X86_PF_USER;
tsk->thread.cr2 = address;
}
static noinline void static noinline void
no_context(struct pt_regs *regs, unsigned long error_code, no_context(struct pt_regs *regs, unsigned long error_code,
unsigned long address, int signal, int si_code) unsigned long address, int signal, int si_code)
@ -639,6 +745,15 @@ no_context(struct pt_regs *regs, unsigned long error_code,
unsigned long flags; unsigned long flags;
int sig; int sig;
if (user_mode(regs)) {
/*
* This is an implicit supervisor-mode access from user
* mode. Bypass all the kernel-mode recovery code and just
* OOPS.
*/
goto oops;
}
/* Are we prepared to handle this kernel fault? */ /* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs, X86_TRAP_PF, error_code, address)) { if (fixup_exception(regs, X86_TRAP_PF, error_code, address)) {
/* /*
@ -656,9 +771,7 @@ no_context(struct pt_regs *regs, unsigned long error_code,
* faulting through the emulate_vsyscall() logic. * faulting through the emulate_vsyscall() logic.
*/ */
if (current->thread.sig_on_uaccess_err && signal) { if (current->thread.sig_on_uaccess_err && signal) {
tsk->thread.trap_nr = X86_TRAP_PF; set_signal_archinfo(address, error_code);
tsk->thread.error_code = error_code | X86_PF_USER;
tsk->thread.cr2 = address;
/* XXX: hwpoison faults will set the wrong code. */ /* XXX: hwpoison faults will set the wrong code. */
force_sig_fault(signal, si_code, (void __user *)address, force_sig_fault(signal, si_code, (void __user *)address,
@ -726,6 +839,7 @@ no_context(struct pt_regs *regs, unsigned long error_code,
if (IS_ENABLED(CONFIG_EFI)) if (IS_ENABLED(CONFIG_EFI))
efi_recover_from_page_fault(address); efi_recover_from_page_fault(address);
oops:
/* /*
* Oops. The kernel tried to access some bad page. We'll have to * Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice: * terminate things with extreme prejudice:
@ -737,10 +851,6 @@ no_context(struct pt_regs *regs, unsigned long error_code,
if (task_stack_end_corrupted(tsk)) if (task_stack_end_corrupted(tsk))
printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
tsk->thread.cr2 = address;
tsk->thread.trap_nr = X86_TRAP_PF;
tsk->thread.error_code = error_code;
sig = SIGKILL; sig = SIGKILL;
if (__die("Oops", regs, error_code)) if (__die("Oops", regs, error_code))
sig = 0; sig = 0;
@ -794,7 +904,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
struct task_struct *tsk = current; struct task_struct *tsk = current;
/* User mode accesses just cause a SIGSEGV */ /* User mode accesses just cause a SIGSEGV */
if (error_code & X86_PF_USER) { if (user_mode(regs) && (error_code & X86_PF_USER)) {
/* /*
* It's possible to have interrupts off here: * It's possible to have interrupts off here:
*/ */
@ -821,9 +931,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
if (likely(show_unhandled_signals)) if (likely(show_unhandled_signals))
show_signal_msg(regs, error_code, address, tsk); show_signal_msg(regs, error_code, address, tsk);
tsk->thread.cr2 = address; set_signal_archinfo(address, error_code);
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_PF;
if (si_code == SEGV_PKUERR) if (si_code == SEGV_PKUERR)
force_sig_pkuerr((void __user *)address, pkey); force_sig_pkuerr((void __user *)address, pkey);
@ -937,9 +1045,7 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
if (is_prefetch(regs, error_code, address)) if (is_prefetch(regs, error_code, address))
return; return;
tsk->thread.cr2 = address; set_signal_archinfo(address, error_code);
tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_PF;
#ifdef CONFIG_MEMORY_FAILURE #ifdef CONFIG_MEMORY_FAILURE
if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { if (fault & (VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
@ -1148,23 +1254,6 @@ static int fault_in_kernel_space(unsigned long address)
return address >= TASK_SIZE_MAX; return address >= TASK_SIZE_MAX;
} }
static inline bool smap_violation(int error_code, struct pt_regs *regs)
{
if (!IS_ENABLED(CONFIG_X86_SMAP))
return false;
if (!static_cpu_has(X86_FEATURE_SMAP))
return false;
if (error_code & X86_PF_USER)
return false;
if (!user_mode(regs) && (regs->flags & X86_EFLAGS_AC))
return false;
return true;
}
/* /*
* Called for all faults where 'address' is part of the kernel address * Called for all faults where 'address' is part of the kernel address
* space. Might get called for faults that originate from *code* that * space. Might get called for faults that originate from *code* that
@ -1230,7 +1319,6 @@ void do_user_addr_fault(struct pt_regs *regs,
unsigned long hw_error_code, unsigned long hw_error_code,
unsigned long address) unsigned long address)
{ {
unsigned long sw_error_code;
struct vm_area_struct *vma; struct vm_area_struct *vma;
struct task_struct *tsk; struct task_struct *tsk;
struct mm_struct *mm; struct mm_struct *mm;
@ -1252,10 +1340,16 @@ void do_user_addr_fault(struct pt_regs *regs,
pgtable_bad(regs, hw_error_code, address); pgtable_bad(regs, hw_error_code, address);
/* /*
* Check for invalid kernel (supervisor) access to user * If SMAP is on, check for invalid kernel (supervisor) access to user
* pages in the user address space. * pages in the user address space. The odd case here is WRUSS,
* which, according to the preliminary documentation, does not respect
* SMAP and will have the USER bit set so, in all cases, SMAP
* enforcement appears to be consistent with the USER bit.
*/ */
if (unlikely(smap_violation(hw_error_code, regs))) { if (unlikely(cpu_feature_enabled(X86_FEATURE_SMAP) &&
!(hw_error_code & X86_PF_USER) &&
!(regs->flags & X86_EFLAGS_AC)))
{
bad_area_nosemaphore(regs, hw_error_code, address); bad_area_nosemaphore(regs, hw_error_code, address);
return; return;
} }
@ -1269,13 +1363,6 @@ void do_user_addr_fault(struct pt_regs *regs,
return; return;
} }
/*
* hw_error_code is literally the "page fault error code" passed to
* the kernel directly from the hardware. But, we will shortly be
* modifying it in software, so give it a new name.
*/
sw_error_code = hw_error_code;
/* /*
* It's safe to allow irq's after cr2 has been saved and the * It's safe to allow irq's after cr2 has been saved and the
* vmalloc fault has been handled. * vmalloc fault has been handled.
@ -1285,26 +1372,6 @@ void do_user_addr_fault(struct pt_regs *regs,
*/ */
if (user_mode(regs)) { if (user_mode(regs)) {
local_irq_enable(); local_irq_enable();
/*
* Up to this point, X86_PF_USER set in hw_error_code
* indicated a user-mode access. But, after this,
* X86_PF_USER in sw_error_code will indicate either
* that, *or* an implicit kernel(supervisor)-mode access
* which originated from user mode.
*/
if (!(hw_error_code & X86_PF_USER)) {
/*
* The CPU was in user mode, but the CPU says
* the fault was not a user-mode access.
* Must be an implicit kernel-mode access,
* which we do not expect to happen in the
* user address space.
*/
pr_warn_once("kernel-mode error from user-mode: %lx\n",
hw_error_code);
sw_error_code |= X86_PF_USER;
}
flags |= FAULT_FLAG_USER; flags |= FAULT_FLAG_USER;
} else { } else {
if (regs->flags & X86_EFLAGS_IF) if (regs->flags & X86_EFLAGS_IF)
@ -1313,9 +1380,9 @@ void do_user_addr_fault(struct pt_regs *regs,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
if (sw_error_code & X86_PF_WRITE) if (hw_error_code & X86_PF_WRITE)
flags |= FAULT_FLAG_WRITE; flags |= FAULT_FLAG_WRITE;
if (sw_error_code & X86_PF_INSTR) if (hw_error_code & X86_PF_INSTR)
flags |= FAULT_FLAG_INSTRUCTION; flags |= FAULT_FLAG_INSTRUCTION;
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
@ -1328,7 +1395,7 @@ void do_user_addr_fault(struct pt_regs *regs,
* The vsyscall page does not have a "real" VMA, so do this * The vsyscall page does not have a "real" VMA, so do this
* emulation before we go searching for VMAs. * emulation before we go searching for VMAs.
*/ */
if ((sw_error_code & X86_PF_INSTR) && is_vsyscall_vaddr(address)) { if ((hw_error_code & X86_PF_INSTR) && is_vsyscall_vaddr(address)) {
if (emulate_vsyscall(regs, address)) if (emulate_vsyscall(regs, address))
return; return;
} }
@ -1344,18 +1411,15 @@ void do_user_addr_fault(struct pt_regs *regs,
* Only do the expensive exception table search when we might be at * Only do the expensive exception table search when we might be at
* risk of a deadlock. This happens if we * risk of a deadlock. This happens if we
* 1. Failed to acquire mmap_sem, and * 1. Failed to acquire mmap_sem, and
* 2. The access did not originate in userspace. Note: either the * 2. The access did not originate in userspace.
* hardware or earlier page fault code may set X86_PF_USER
* in sw_error_code.
*/ */
if (unlikely(!down_read_trylock(&mm->mmap_sem))) { if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
if (!(sw_error_code & X86_PF_USER) && if (!user_mode(regs) && !search_exception_tables(regs->ip)) {
!search_exception_tables(regs->ip)) {
/* /*
* Fault from code in kernel from * Fault from code in kernel from
* which we do not expect faults. * which we do not expect faults.
*/ */
bad_area_nosemaphore(regs, sw_error_code, address); bad_area_nosemaphore(regs, hw_error_code, address);
return; return;
} }
retry: retry:
@ -1371,29 +1435,17 @@ retry:
vma = find_vma(mm, address); vma = find_vma(mm, address);
if (unlikely(!vma)) { if (unlikely(!vma)) {
bad_area(regs, sw_error_code, address); bad_area(regs, hw_error_code, address);
return; return;
} }
if (likely(vma->vm_start <= address)) if (likely(vma->vm_start <= address))
goto good_area; goto good_area;
if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
bad_area(regs, sw_error_code, address); bad_area(regs, hw_error_code, address);
return; return;
} }
if (sw_error_code & X86_PF_USER) {
/*
* Accessing the stack below %sp is always a bug.
* The large cushion allows instructions like enter
* and pusha to work. ("enter $65535, $31" pushes
* 32 pointers and then decrements %sp by 65535.)
*/
if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
bad_area(regs, sw_error_code, address);
return;
}
}
if (unlikely(expand_stack(vma, address))) { if (unlikely(expand_stack(vma, address))) {
bad_area(regs, sw_error_code, address); bad_area(regs, hw_error_code, address);
return; return;
} }
@ -1402,8 +1454,8 @@ retry:
* we can handle it.. * we can handle it..
*/ */
good_area: good_area:
if (unlikely(access_error(sw_error_code, vma))) { if (unlikely(access_error(hw_error_code, vma))) {
bad_area_access_error(regs, sw_error_code, address, vma); bad_area_access_error(regs, hw_error_code, address, vma);
return; return;
} }
@ -1442,13 +1494,13 @@ good_area:
return; return;
/* Not returning to user mode? Handle exceptions or die: */ /* Not returning to user mode? Handle exceptions or die: */
no_context(regs, sw_error_code, address, SIGBUS, BUS_ADRERR); no_context(regs, hw_error_code, address, SIGBUS, BUS_ADRERR);
return; return;
} }
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
if (unlikely(fault & VM_FAULT_ERROR)) { if (unlikely(fault & VM_FAULT_ERROR)) {
mm_fault_error(regs, sw_error_code, address, fault); mm_fault_error(regs, hw_error_code, address, fault);
return; return;
} }

30
arch/x86/mm/init_64.c
View File

@ -432,7 +432,7 @@ phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end,
E820_TYPE_RAM) && E820_TYPE_RAM) &&
!e820__mapped_any(paddr & PAGE_MASK, paddr_next, !e820__mapped_any(paddr & PAGE_MASK, paddr_next,
E820_TYPE_RESERVED_KERN)) E820_TYPE_RESERVED_KERN))
set_pte(pte, __pte(0)); set_pte_safe(pte, __pte(0));
continue; continue;
} }
@ -452,7 +452,7 @@ phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end,
pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr, pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr,
pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte); pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte);
pages++; pages++;
set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot)); set_pte_safe(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE; paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE;
} }
@ -487,7 +487,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end,
E820_TYPE_RAM) && E820_TYPE_RAM) &&
!e820__mapped_any(paddr & PMD_MASK, paddr_next, !e820__mapped_any(paddr & PMD_MASK, paddr_next,
E820_TYPE_RESERVED_KERN)) E820_TYPE_RESERVED_KERN))
set_pmd(pmd, __pmd(0)); set_pmd_safe(pmd, __pmd(0));
continue; continue;
} }
@ -524,7 +524,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end,
if (page_size_mask & (1<<PG_LEVEL_2M)) { if (page_size_mask & (1<<PG_LEVEL_2M)) {
pages++; pages++;
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
set_pte((pte_t *)pmd, set_pte_safe((pte_t *)pmd,
pfn_pte((paddr & PMD_MASK) >> PAGE_SHIFT, pfn_pte((paddr & PMD_MASK) >> PAGE_SHIFT,
__pgprot(pgprot_val(prot) | _PAGE_PSE))); __pgprot(pgprot_val(prot) | _PAGE_PSE)));
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
@ -536,7 +536,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end,
paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot); paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot);
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
pmd_populate_kernel(&init_mm, pmd, pte); pmd_populate_kernel_safe(&init_mm, pmd, pte);
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
} }
update_page_count(PG_LEVEL_2M, pages); update_page_count(PG_LEVEL_2M, pages);
@ -573,7 +573,7 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
E820_TYPE_RAM) && E820_TYPE_RAM) &&
!e820__mapped_any(paddr & PUD_MASK, paddr_next, !e820__mapped_any(paddr & PUD_MASK, paddr_next,
E820_TYPE_RESERVED_KERN)) E820_TYPE_RESERVED_KERN))
set_pud(pud, __pud(0)); set_pud_safe(pud, __pud(0));
continue; continue;
} }
@ -584,7 +584,6 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
paddr_end, paddr_end,
page_size_mask, page_size_mask,
prot); prot);
__flush_tlb_all();
continue; continue;
} }
/* /*
@ -611,7 +610,7 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
if (page_size_mask & (1<<PG_LEVEL_1G)) { if (page_size_mask & (1<<PG_LEVEL_1G)) {
pages++; pages++;
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
set_pte((pte_t *)pud, set_pte_safe((pte_t *)pud,
pfn_pte((paddr & PUD_MASK) >> PAGE_SHIFT, pfn_pte((paddr & PUD_MASK) >> PAGE_SHIFT,
PAGE_KERNEL_LARGE)); PAGE_KERNEL_LARGE));
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
@ -624,10 +623,9 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
page_size_mask, prot); page_size_mask, prot);
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
pud_populate(&init_mm, pud, pmd); pud_populate_safe(&init_mm, pud, pmd);
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
} }
__flush_tlb_all();
update_page_count(PG_LEVEL_1G, pages); update_page_count(PG_LEVEL_1G, pages);
@ -659,7 +657,7 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
E820_TYPE_RAM) && E820_TYPE_RAM) &&
!e820__mapped_any(paddr & P4D_MASK, paddr_next, !e820__mapped_any(paddr & P4D_MASK, paddr_next,
E820_TYPE_RESERVED_KERN)) E820_TYPE_RESERVED_KERN))
set_p4d(p4d, __p4d(0)); set_p4d_safe(p4d, __p4d(0));
continue; continue;
} }
@ -668,7 +666,6 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
paddr_last = phys_pud_init(pud, paddr, paddr_last = phys_pud_init(pud, paddr,
paddr_end, paddr_end,
page_size_mask); page_size_mask);
__flush_tlb_all();
continue; continue;
} }
@ -677,10 +674,9 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
page_size_mask); page_size_mask);
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
p4d_populate(&init_mm, p4d, pud); p4d_populate_safe(&init_mm, p4d, pud);
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
} }
__flush_tlb_all();
return paddr_last; return paddr_last;
} }
@ -723,9 +719,9 @@ kernel_physical_mapping_init(unsigned long paddr_start,
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
if (pgtable_l5_enabled()) if (pgtable_l5_enabled())
pgd_populate(&init_mm, pgd, p4d); pgd_populate_safe(&init_mm, pgd, p4d);
else else
p4d_populate(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d); p4d_populate_safe(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d);
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
pgd_changed = true; pgd_changed = true;
} }
@ -733,8 +729,6 @@ kernel_physical_mapping_init(unsigned long paddr_start,
if (pgd_changed) if (pgd_changed)
sync_global_pgds(vaddr_start, vaddr_end - 1); sync_global_pgds(vaddr_start, vaddr_end - 1);
__flush_tlb_all();
return paddr_last; return paddr_last;
} }

2
arch/x86/mm/mm_internal.h
View File

@ -19,4 +19,6 @@ extern int after_bootmem;
void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache); void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache);
extern unsigned long tlb_single_page_flush_ceiling;
#endif /* __X86_MM_INTERNAL_H */ #endif /* __X86_MM_INTERNAL_H */

31
arch/x86/mm/pageattr-test.c
View File

@ -23,7 +23,8 @@
static __read_mostly int print = 1; static __read_mostly int print = 1;
enum { enum {
NTEST = 400, NTEST = 3 * 100,
NPAGES = 100,
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
LPS = (1 << PMD_SHIFT), LPS = (1 << PMD_SHIFT),
#elif defined(CONFIG_X86_PAE) #elif defined(CONFIG_X86_PAE)
@ -110,6 +111,9 @@ static int print_split(struct split_state *s)
static unsigned long addr[NTEST]; static unsigned long addr[NTEST];
static unsigned int len[NTEST]; static unsigned int len[NTEST];
static struct page *pages[NPAGES];
static unsigned long addrs[NPAGES];
/* Change the global bit on random pages in the direct mapping */ /* Change the global bit on random pages in the direct mapping */
static int pageattr_test(void) static int pageattr_test(void)
{ {
@ -120,7 +124,6 @@ static int pageattr_test(void)
unsigned int level; unsigned int level;
int i, k; int i, k;
int err; int err;
unsigned long test_addr;
if (print) if (print)
printk(KERN_INFO "CPA self-test:\n"); printk(KERN_INFO "CPA self-test:\n");
@ -137,7 +140,7 @@ static int pageattr_test(void)
unsigned long pfn = prandom_u32() % max_pfn_mapped; unsigned long pfn = prandom_u32() % max_pfn_mapped;
addr[i] = (unsigned long)__va(pfn << PAGE_SHIFT); addr[i] = (unsigned long)__va(pfn << PAGE_SHIFT);
len[i] = prandom_u32() % 100; len[i] = prandom_u32() % NPAGES;
len[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1); len[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1);
if (len[i] == 0) if (len[i] == 0)
@ -167,14 +170,29 @@ static int pageattr_test(void)
break; break;
} }
__set_bit(pfn + k, bm); __set_bit(pfn + k, bm);
addrs[k] = addr[i] + k*PAGE_SIZE;
pages[k] = pfn_to_page(pfn + k);
} }
if (!addr[i] || !pte || !k) { if (!addr[i] || !pte || !k) {
addr[i] = 0; addr[i] = 0;
continue; continue;
} }
test_addr = addr[i]; switch (i % 3) {
err = change_page_attr_set(&test_addr, len[i], PAGE_CPA_TEST, 0); case 0:
err = change_page_attr_set(&addr[i], len[i], PAGE_CPA_TEST, 0);
break;
case 1:
err = change_page_attr_set(addrs, len[1], PAGE_CPA_TEST, 1);
break;
case 2:
err = cpa_set_pages_array(pages, len[i], PAGE_CPA_TEST);
break;
}
if (err < 0) { if (err < 0) {
printk(KERN_ERR "CPA %d failed %d\n", i, err); printk(KERN_ERR "CPA %d failed %d\n", i, err);
failed++; failed++;
@ -206,8 +224,7 @@ static int pageattr_test(void)
failed++; failed++;
continue; continue;
} }
test_addr = addr[i]; err = change_page_attr_clear(&addr[i], len[i], PAGE_CPA_TEST, 0);
err = change_page_attr_clear(&test_addr, len[i], PAGE_CPA_TEST, 0);
if (err < 0) { if (err < 0) {
printk(KERN_ERR "CPA reverting failed: %d\n", err); printk(KERN_ERR "CPA reverting failed: %d\n", err);
failed++; failed++;

277
arch/x86/mm/pageattr.c
View File

@ -26,6 +26,8 @@
#include <asm/pat.h> #include <asm/pat.h>
#include <asm/set_memory.h> #include <asm/set_memory.h>
#include "mm_internal.h"
/* /*
* The current flushing context - we pass it instead of 5 arguments: * The current flushing context - we pass it instead of 5 arguments:
*/ */
@ -35,11 +37,11 @@ struct cpa_data {
pgprot_t mask_set; pgprot_t mask_set;
pgprot_t mask_clr; pgprot_t mask_clr;
unsigned long numpages; unsigned long numpages;
int flags; unsigned long curpage;
unsigned long pfn; unsigned long pfn;
unsigned force_split : 1, unsigned int flags;
unsigned int force_split : 1,
force_static_prot : 1; force_static_prot : 1;
int curpage;
struct page **pages; struct page **pages;
}; };
@ -228,19 +230,28 @@ static bool __cpa_pfn_in_highmap(unsigned long pfn)
#endif #endif
static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
{
if (cpa->flags & CPA_PAGES_ARRAY) {
struct page *page = cpa->pages[idx];
if (unlikely(PageHighMem(page)))
return 0;
return (unsigned long)page_address(page);
}
if (cpa->flags & CPA_ARRAY)
return cpa->vaddr[idx];
return *cpa->vaddr + idx * PAGE_SIZE;
}
/* /*
* Flushing functions * Flushing functions
*/ */
/** static void clflush_cache_range_opt(void *vaddr, unsigned int size)
* clflush_cache_range - flush a cache range with clflush
* @vaddr: virtual start address
* @size: number of bytes to flush
*
* clflushopt is an unordered instruction which needs fencing with mfence or
* sfence to avoid ordering issues.
*/
void clflush_cache_range(void *vaddr, unsigned int size)
{ {
const unsigned long clflush_size = boot_cpu_data.x86_clflush_size; const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1)); void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
@ -249,11 +260,22 @@ void clflush_cache_range(void *vaddr, unsigned int size)
if (p >= vend) if (p >= vend)
return; return;
mb();
for (; p < vend; p += clflush_size) for (; p < vend; p += clflush_size)
clflushopt(p); clflushopt(p);
}
/**
* clflush_cache_range - flush a cache range with clflush
* @vaddr: virtual start address
* @size: number of bytes to flush
*
* CLFLUSHOPT is an unordered instruction which needs fencing with MFENCE or
* SFENCE to avoid ordering issues.
*/
void clflush_cache_range(void *vaddr, unsigned int size)
{
mb();
clflush_cache_range_opt(vaddr, size);
mb(); mb();
} }
EXPORT_SYMBOL_GPL(clflush_cache_range); EXPORT_SYMBOL_GPL(clflush_cache_range);
@ -285,87 +307,49 @@ static void cpa_flush_all(unsigned long cache)
on_each_cpu(__cpa_flush_all, (void *) cache, 1); on_each_cpu(__cpa_flush_all, (void *) cache, 1);
} }
static bool __inv_flush_all(int cache) void __cpa_flush_tlb(void *data)
{ {
struct cpa_data *cpa = data;
unsigned int i;
for (i = 0; i < cpa->numpages; i++)
__flush_tlb_one_kernel(__cpa_addr(cpa, i));
}
static void cpa_flush(struct cpa_data *data, int cache)
{
struct cpa_data *cpa = data;
unsigned int i;
BUG_ON(irqs_disabled() && !early_boot_irqs_disabled); BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) { if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
cpa_flush_all(cache); cpa_flush_all(cache);
return true; return;
} }
return false; if (cpa->numpages <= tlb_single_page_flush_ceiling)
} on_each_cpu(__cpa_flush_tlb, cpa, 1);
else
static void cpa_flush_range(unsigned long start, int numpages, int cache) flush_tlb_all();
{
unsigned int i, level;
unsigned long addr;
WARN_ON(PAGE_ALIGN(start) != start);
if (__inv_flush_all(cache))
return;
flush_tlb_kernel_range(start, start + PAGE_SIZE * numpages);
if (!cache) if (!cache)
return; return;
/* mb();
* We only need to flush on one CPU, for (i = 0; i < cpa->numpages; i++) {
* clflush is a MESI-coherent instruction that unsigned long addr = __cpa_addr(cpa, i);
* will cause all other CPUs to flush the same unsigned int level;
* cachelines:
*/
for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
pte_t *pte = lookup_address(addr, &level); pte_t *pte = lookup_address(addr, &level);
/* /*
* Only flush present addresses: * Only flush present addresses:
*/ */
if (pte && (pte_val(*pte) & _PAGE_PRESENT)) if (pte && (pte_val(*pte) & _PAGE_PRESENT))
clflush_cache_range((void *) addr, PAGE_SIZE); clflush_cache_range_opt((void *)addr, PAGE_SIZE);
}
}
static void cpa_flush_array(unsigned long baddr, unsigned long *start,
int numpages, int cache,
int in_flags, struct page **pages)
{
unsigned int i, level;
if (__inv_flush_all(cache))
return;
flush_tlb_all();
if (!cache)
return;
/*
* We only need to flush on one CPU,
* clflush is a MESI-coherent instruction that
* will cause all other CPUs to flush the same
* cachelines:
*/
for (i = 0; i < numpages; i++) {
unsigned long addr;
pte_t *pte;
if (in_flags & CPA_PAGES_ARRAY)
addr = (unsigned long)page_address(pages[i]);
else
addr = start[i];
pte = lookup_address(addr, &level);
/*
* Only flush present addresses:
*/
if (pte && (pte_val(*pte) & _PAGE_PRESENT))
clflush_cache_range((void *)addr, PAGE_SIZE);
} }
mb();
} }
static bool overlaps(unsigned long r1_start, unsigned long r1_end, static bool overlaps(unsigned long r1_start, unsigned long r1_end,
@ -1476,15 +1460,7 @@ static int __change_page_attr(struct cpa_data *cpa, int primary)
unsigned int level; unsigned int level;
pte_t *kpte, old_pte; pte_t *kpte, old_pte;
if (cpa->flags & CPA_PAGES_ARRAY) { address = __cpa_addr(cpa, cpa->curpage);
struct page *page = cpa->pages[cpa->curpage];
if (unlikely(PageHighMem(page)))
return 0;
address = (unsigned long)page_address(page);
} else if (cpa->flags & CPA_ARRAY)
address = cpa->vaddr[cpa->curpage];
else
address = *cpa->vaddr;
repeat: repeat:
kpte = _lookup_address_cpa(cpa, address, &level); kpte = _lookup_address_cpa(cpa, address, &level);
if (!kpte) if (!kpte)
@ -1565,22 +1541,14 @@ static int cpa_process_alias(struct cpa_data *cpa)
* No need to redo, when the primary call touched the direct * No need to redo, when the primary call touched the direct
* mapping already: * mapping already:
*/ */
if (cpa->flags & CPA_PAGES_ARRAY) { vaddr = __cpa_addr(cpa, cpa->curpage);
struct page *page = cpa->pages[cpa->curpage];
if (unlikely(PageHighMem(page)))
return 0;
vaddr = (unsigned long)page_address(page);
} else if (cpa->flags & CPA_ARRAY)
vaddr = cpa->vaddr[cpa->curpage];
else
vaddr = *cpa->vaddr;
if (!(within(vaddr, PAGE_OFFSET, if (!(within(vaddr, PAGE_OFFSET,
PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) { PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
alias_cpa = *cpa; alias_cpa = *cpa;
alias_cpa.vaddr = &laddr; alias_cpa.vaddr = &laddr;
alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY); alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
alias_cpa.curpage = 0;
ret = __change_page_attr_set_clr(&alias_cpa, 0); ret = __change_page_attr_set_clr(&alias_cpa, 0);
if (ret) if (ret)
@ -1600,6 +1568,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
alias_cpa = *cpa; alias_cpa = *cpa;
alias_cpa.vaddr = &temp_cpa_vaddr; alias_cpa.vaddr = &temp_cpa_vaddr;
alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY); alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
alias_cpa.curpage = 0;
/* /*
* The high mapping range is imprecise, so ignore the * The high mapping range is imprecise, so ignore the
@ -1615,14 +1584,15 @@ static int cpa_process_alias(struct cpa_data *cpa)
static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
{ {
unsigned long numpages = cpa->numpages; unsigned long numpages = cpa->numpages;
int ret; unsigned long rempages = numpages;
int ret = 0;
while (numpages) { while (rempages) {
/* /*
* Store the remaining nr of pages for the large page * Store the remaining nr of pages for the large page
* preservation check. * preservation check.
*/ */
cpa->numpages = numpages; cpa->numpages = rempages;
/* for array changes, we can't use large page */ /* for array changes, we can't use large page */
if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
cpa->numpages = 1; cpa->numpages = 1;
@ -1633,12 +1603,12 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
if (!debug_pagealloc_enabled()) if (!debug_pagealloc_enabled())
spin_unlock(&cpa_lock); spin_unlock(&cpa_lock);
if (ret) if (ret)
return ret; goto out;
if (checkalias) { if (checkalias) {
ret = cpa_process_alias(cpa); ret = cpa_process_alias(cpa);
if (ret) if (ret)
return ret; goto out;
} }
/* /*
@ -1646,15 +1616,15 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
* CPA operation. Either a large page has been * CPA operation. Either a large page has been
* preserved or a single page update happened. * preserved or a single page update happened.
*/ */
BUG_ON(cpa->numpages > numpages || !cpa->numpages); BUG_ON(cpa->numpages > rempages || !cpa->numpages);
numpages -= cpa->numpages; rempages -= cpa->numpages;
if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) cpa->curpage += cpa->numpages;
cpa->curpage++;
else
*cpa->vaddr += cpa->numpages * PAGE_SIZE;
} }
return 0;
out:
/* Restore the original numpages */
cpa->numpages = numpages;
return ret;
} }
/* /*
@ -1687,7 +1657,6 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
{ {
struct cpa_data cpa; struct cpa_data cpa;
int ret, cache, checkalias; int ret, cache, checkalias;
unsigned long baddr = 0;
memset(&cpa, 0, sizeof(cpa)); memset(&cpa, 0, sizeof(cpa));
@ -1721,11 +1690,6 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
*/ */
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
} }
/*
* Save address for cache flush. *addr is modified in the call
* to __change_page_attr_set_clr() below.
*/
baddr = make_addr_canonical_again(*addr);
} }
/* Must avoid aliasing mappings in the highmem code */ /* Must avoid aliasing mappings in the highmem code */
@ -1773,13 +1737,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
goto out; goto out;
} }
if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { cpa_flush(&cpa, cache);
cpa_flush_array(baddr, addr, numpages, cache,
cpa.flags, pages);
} else {
cpa_flush_range(baddr, numpages, cache);
}
out: out:
return ret; return ret;
} }
@ -1850,14 +1808,14 @@ out_err:
} }
EXPORT_SYMBOL(set_memory_uc); EXPORT_SYMBOL(set_memory_uc);
static int _set_memory_array(unsigned long *addr, int addrinarray, static int _set_memory_array(unsigned long *addr, int numpages,
enum page_cache_mode new_type) enum page_cache_mode new_type)
{ {
enum page_cache_mode set_type; enum page_cache_mode set_type;
int i, j; int i, j;
int ret; int ret;
for (i = 0; i < addrinarray; i++) { for (i = 0; i < numpages; i++) {
ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE, ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
new_type, NULL); new_type, NULL);
if (ret) if (ret)
@ -1868,11 +1826,11 @@ static int _set_memory_array(unsigned long *addr, int addrinarray,
set_type = (new_type == _PAGE_CACHE_MODE_WC) ? set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
_PAGE_CACHE_MODE_UC_MINUS : new_type; _PAGE_CACHE_MODE_UC_MINUS : new_type;
ret = change_page_attr_set(addr, addrinarray, ret = change_page_attr_set(addr, numpages,
cachemode2pgprot(set_type), 1); cachemode2pgprot(set_type), 1);
if (!ret && new_type == _PAGE_CACHE_MODE_WC) if (!ret && new_type == _PAGE_CACHE_MODE_WC)
ret = change_page_attr_set_clr(addr, addrinarray, ret = change_page_attr_set_clr(addr, numpages,
cachemode2pgprot( cachemode2pgprot(
_PAGE_CACHE_MODE_WC), _PAGE_CACHE_MODE_WC),
__pgprot(_PAGE_CACHE_MASK), __pgprot(_PAGE_CACHE_MASK),
@ -1889,36 +1847,34 @@ out_free:
return ret; return ret;
} }
int set_memory_array_uc(unsigned long *addr, int addrinarray) int set_memory_array_uc(unsigned long *addr, int numpages)
{ {
return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_UC_MINUS); return _set_memory_array(addr, numpages, _PAGE_CACHE_MODE_UC_MINUS);
} }
EXPORT_SYMBOL(set_memory_array_uc); EXPORT_SYMBOL(set_memory_array_uc);
int set_memory_array_wc(unsigned long *addr, int addrinarray) int set_memory_array_wc(unsigned long *addr, int numpages)
{ {
return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WC); return _set_memory_array(addr, numpages, _PAGE_CACHE_MODE_WC);
} }
EXPORT_SYMBOL(set_memory_array_wc); EXPORT_SYMBOL(set_memory_array_wc);
int set_memory_array_wt(unsigned long *addr, int addrinarray) int set_memory_array_wt(unsigned long *addr, int numpages)
{ {
return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WT); return _set_memory_array(addr, numpages, _PAGE_CACHE_MODE_WT);
} }
EXPORT_SYMBOL_GPL(set_memory_array_wt); EXPORT_SYMBOL_GPL(set_memory_array_wt);
int _set_memory_wc(unsigned long addr, int numpages) int _set_memory_wc(unsigned long addr, int numpages)
{ {
int ret; int ret;
unsigned long addr_copy = addr;
ret = change_page_attr_set(&addr, numpages, ret = change_page_attr_set(&addr, numpages,
cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS), cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
0); 0);
if (!ret) { if (!ret) {
ret = change_page_attr_set_clr(&addr_copy, numpages, ret = change_page_attr_set_clr(&addr, numpages,
cachemode2pgprot( cachemode2pgprot(_PAGE_CACHE_MODE_WC),
_PAGE_CACHE_MODE_WC),
__pgprot(_PAGE_CACHE_MASK), __pgprot(_PAGE_CACHE_MASK),
0, 0, NULL); 0, 0, NULL);
} }
@ -1985,18 +1941,18 @@ int set_memory_wb(unsigned long addr, int numpages)
} }
EXPORT_SYMBOL(set_memory_wb); EXPORT_SYMBOL(set_memory_wb);
int set_memory_array_wb(unsigned long *addr, int addrinarray) int set_memory_array_wb(unsigned long *addr, int numpages)
{ {
int i; int i;
int ret; int ret;
/* WB cache mode is hard wired to all cache attribute bits being 0 */ /* WB cache mode is hard wired to all cache attribute bits being 0 */
ret = change_page_attr_clear(addr, addrinarray, ret = change_page_attr_clear(addr, numpages,
__pgprot(_PAGE_CACHE_MASK), 1); __pgprot(_PAGE_CACHE_MASK), 1);
if (ret) if (ret)
return ret; return ret;
for (i = 0; i < addrinarray; i++) for (i = 0; i < numpages; i++)
free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE); free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
return 0; return 0;
@ -2066,7 +2022,6 @@ int set_memory_global(unsigned long addr, int numpages)
static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc) static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
{ {
struct cpa_data cpa; struct cpa_data cpa;
unsigned long start;
int ret; int ret;
/* Nothing to do if memory encryption is not active */ /* Nothing to do if memory encryption is not active */
@ -2077,8 +2032,6 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr)) if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
addr &= PAGE_MASK; addr &= PAGE_MASK;
start = addr;
memset(&cpa, 0, sizeof(cpa)); memset(&cpa, 0, sizeof(cpa));
cpa.vaddr = &addr; cpa.vaddr = &addr;
cpa.numpages = numpages; cpa.numpages = numpages;
@ -2093,18 +2046,18 @@ static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
/* /*
* Before changing the encryption attribute, we need to flush caches. * Before changing the encryption attribute, we need to flush caches.
*/ */
cpa_flush_range(start, numpages, 1); cpa_flush(&cpa, 1);
ret = __change_page_attr_set_clr(&cpa, 1); ret = __change_page_attr_set_clr(&cpa, 1);
/* /*
* After changing the encryption attribute, we need to flush TLBs * After changing the encryption attribute, we need to flush TLBs again
* again in case any speculative TLB caching occurred (but no need * in case any speculative TLB caching occurred (but no need to flush
* to flush caches again). We could just use cpa_flush_all(), but * caches again). We could just use cpa_flush_all(), but in case TLB
* in case TLB flushing gets optimized in the cpa_flush_range() * flushing gets optimized in the cpa_flush() path use the same logic
* path use the same logic as above. * as above.
*/ */
cpa_flush_range(start, numpages, 0); cpa_flush(&cpa, 0);
return ret; return ret;
} }
@ -2129,7 +2082,7 @@ int set_pages_uc(struct page *page, int numpages)
} }
EXPORT_SYMBOL(set_pages_uc); EXPORT_SYMBOL(set_pages_uc);
static int _set_pages_array(struct page **pages, int addrinarray, static int _set_pages_array(struct page **pages, int numpages,
enum page_cache_mode new_type) enum page_cache_mode new_type)
{ {
unsigned long start; unsigned long start;
@ -2139,7 +2092,7 @@ static int _set_pages_array(struct page **pages, int addrinarray,
int free_idx; int free_idx;
int ret; int ret;
for (i = 0; i < addrinarray; i++) { for (i = 0; i < numpages; i++) {
if (PageHighMem(pages[i])) if (PageHighMem(pages[i]))
continue; continue;
start = page_to_pfn(pages[i]) << PAGE_SHIFT; start = page_to_pfn(pages[i]) << PAGE_SHIFT;
@ -2152,10 +2105,10 @@ static int _set_pages_array(struct page **pages, int addrinarray,
set_type = (new_type == _PAGE_CACHE_MODE_WC) ? set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
_PAGE_CACHE_MODE_UC_MINUS : new_type; _PAGE_CACHE_MODE_UC_MINUS : new_type;
ret = cpa_set_pages_array(pages, addrinarray, ret = cpa_set_pages_array(pages, numpages,
cachemode2pgprot(set_type)); cachemode2pgprot(set_type));
if (!ret && new_type == _PAGE_CACHE_MODE_WC) if (!ret && new_type == _PAGE_CACHE_MODE_WC)
ret = change_page_attr_set_clr(NULL, addrinarray, ret = change_page_attr_set_clr(NULL, numpages,
cachemode2pgprot( cachemode2pgprot(
_PAGE_CACHE_MODE_WC), _PAGE_CACHE_MODE_WC),
__pgprot(_PAGE_CACHE_MASK), __pgprot(_PAGE_CACHE_MASK),
@ -2175,21 +2128,21 @@ err_out:
return -EINVAL; return -EINVAL;
} }
int set_pages_array_uc(struct page **pages, int addrinarray) int set_pages_array_uc(struct page **pages, int numpages)
{ {
return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_UC_MINUS); return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_UC_MINUS);
} }
EXPORT_SYMBOL(set_pages_array_uc); EXPORT_SYMBOL(set_pages_array_uc);
int set_pages_array_wc(struct page **pages, int addrinarray) int set_pages_array_wc(struct page **pages, int numpages)
{ {
return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WC); return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WC);
} }
EXPORT_SYMBOL(set_pages_array_wc); EXPORT_SYMBOL(set_pages_array_wc);
int set_pages_array_wt(struct page **pages, int addrinarray) int set_pages_array_wt(struct page **pages, int numpages)
{ {
return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WT); return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WT);
} }
EXPORT_SYMBOL_GPL(set_pages_array_wt); EXPORT_SYMBOL_GPL(set_pages_array_wt);
@ -2201,7 +2154,7 @@ int set_pages_wb(struct page *page, int numpages)
} }
EXPORT_SYMBOL(set_pages_wb); EXPORT_SYMBOL(set_pages_wb);
int set_pages_array_wb(struct page **pages, int addrinarray) int set_pages_array_wb(struct page **pages, int numpages)
{ {
int retval; int retval;
unsigned long start; unsigned long start;
@ -2209,12 +2162,12 @@ int set_pages_array_wb(struct page **pages, int addrinarray)
int i; int i;
/* WB cache mode is hard wired to all cache attribute bits being 0 */ /* WB cache mode is hard wired to all cache attribute bits being 0 */
retval = cpa_clear_pages_array(pages, addrinarray, retval = cpa_clear_pages_array(pages, numpages,
__pgprot(_PAGE_CACHE_MASK)); __pgprot(_PAGE_CACHE_MASK));
if (retval) if (retval)
return retval; return retval;
for (i = 0; i < addrinarray; i++) { for (i = 0; i < numpages; i++) {
if (PageHighMem(pages[i])) if (PageHighMem(pages[i]))
continue; continue;
start = page_to_pfn(pages[i]) << PAGE_SHIFT; start = page_to_pfn(pages[i]) << PAGE_SHIFT;

4
arch/x86/mm/tlb.c
View File

@ -15,6 +15,8 @@
#include <asm/apic.h> #include <asm/apic.h>
#include <asm/uv/uv.h> #include <asm/uv/uv.h>
#include "mm_internal.h"
/* /*
* TLB flushing, formerly SMP-only * TLB flushing, formerly SMP-only
* c/o Linus Torvalds. * c/o Linus Torvalds.
@ -721,7 +723,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
* *
* This is in units of pages. * This is in units of pages.
*/ */
static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long end, unsigned int stride_shift, unsigned long end, unsigned int stride_shift,

1
include/asm-generic/5level-fixup.h
View File

@ -26,6 +26,7 @@
#define p4d_clear(p4d) pgd_clear(p4d) #define p4d_clear(p4d) pgd_clear(p4d)
#define p4d_val(p4d) pgd_val(p4d) #define p4d_val(p4d) pgd_val(p4d)
#define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud) #define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud)
#define p4d_populate_safe(mm, p4d, pud) pgd_populate(mm, p4d, pud)
#define p4d_page(p4d) pgd_page(p4d) #define p4d_page(p4d) pgd_page(p4d)
#define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d) #define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d)

1
include/asm-generic/pgtable-nop4d-hack.h
View File

@ -31,6 +31,7 @@ static inline void pgd_clear(pgd_t *pgd) { }
#define pud_ERROR(pud) (pgd_ERROR((pud).pgd)) #define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
#define pgd_populate(mm, pgd, pud) do { } while (0) #define pgd_populate(mm, pgd, pud) do { } while (0)
#define pgd_populate_safe(mm, pgd, pud) do { } while (0)
/* /*
* (puds are folded into pgds so this doesn't get actually called, * (puds are folded into pgds so this doesn't get actually called,
* but the define is needed for a generic inline function.) * but the define is needed for a generic inline function.)

1
include/asm-generic/pgtable-nop4d.h
View File

@ -26,6 +26,7 @@ static inline void pgd_clear(pgd_t *pgd) { }
#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd)) #define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
#define pgd_populate(mm, pgd, p4d) do { } while (0) #define pgd_populate(mm, pgd, p4d) do { } while (0)
#define pgd_populate_safe(mm, pgd, p4d) do { } while (0)
/* /*
* (p4ds are folded into pgds so this doesn't get actually called, * (p4ds are folded into pgds so this doesn't get actually called,
* but the define is needed for a generic inline function.) * but the define is needed for a generic inline function.)

1
include/asm-generic/pgtable-nopud.h
View File

@ -35,6 +35,7 @@ static inline void p4d_clear(p4d_t *p4d) { }
#define pud_ERROR(pud) (p4d_ERROR((pud).p4d)) #define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
#define p4d_populate(mm, p4d, pud) do { } while (0) #define p4d_populate(mm, p4d, pud) do { } while (0)
#define p4d_populate_safe(mm, p4d, pud) do { } while (0)
/* /*
* (puds are folded into p4ds so this doesn't get actually called, * (puds are folded into p4ds so this doesn't get actually called,
* but the define is needed for a generic inline function.) * but the define is needed for a generic inline function.)

66
include/asm-generic/pgtable.h
View File

@ -375,7 +375,6 @@ static inline int pte_unused(pte_t pte)
#endif #endif
#ifndef __HAVE_ARCH_PMD_SAME #ifndef __HAVE_ARCH_PMD_SAME
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
{ {
return pmd_val(pmd_a) == pmd_val(pmd_b); return pmd_val(pmd_a) == pmd_val(pmd_b);
@ -385,21 +384,60 @@ static inline int pud_same(pud_t pud_a, pud_t pud_b)
{ {
return pud_val(pud_a) == pud_val(pud_b); return pud_val(pud_a) == pud_val(pud_b);
} }
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
{
BUILD_BUG();
return 0;
}
static inline int pud_same(pud_t pud_a, pud_t pud_b)
{
BUILD_BUG();
return 0;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif #endif
#ifndef __HAVE_ARCH_P4D_SAME
static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b)
{
return p4d_val(p4d_a) == p4d_val(p4d_b);
}
#endif
#ifndef __HAVE_ARCH_PGD_SAME
static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b)
{
return pgd_val(pgd_a) == pgd_val(pgd_b);
}
#endif
/*
* Use set_p*_safe(), and elide TLB flushing, when confident that *no*
* TLB flush will be required as a result of the "set". For example, use
* in scenarios where it is known ahead of time that the routine is
* setting non-present entries, or re-setting an existing entry to the
* same value. Otherwise, use the typical "set" helpers and flush the
* TLB.
*/
#define set_pte_safe(ptep, pte) \
({ \
WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
set_pte(ptep, pte); \
})
#define set_pmd_safe(pmdp, pmd) \
({ \
WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
set_pmd(pmdp, pmd); \
})
#define set_pud_safe(pudp, pud) \
({ \
WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
set_pud(pudp, pud); \
})
#define set_p4d_safe(p4dp, p4d) \
({ \
WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
set_p4d(p4dp, p4d); \
})
#define set_pgd_safe(pgdp, pgd) \
({ \
WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
set_pgd(pgdp, pgd); \
})
#ifndef __HAVE_ARCH_DO_SWAP_PAGE #ifndef __HAVE_ARCH_DO_SWAP_PAGE
/* /*
* Some architectures support metadata associated with a page. When a * Some architectures support metadata associated with a page. When a