2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 1991, 1992 Linus Torvalds
|
2008-07-02 07:29:44 +08:00
|
|
|
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
|
* Pentium III FXSR, SSE support
|
|
|
|
|
* Gareth Hughes <gareth@valinux.com>, May 2000
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/*
|
2008-10-04 05:17:11 +08:00
|
|
|
* Handle hardware traps and faults.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2012-05-22 10:50:07 +08:00
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
|
2008-02-26 18:15:50 +08:00
|
|
|
#include <linux/interrupt.h>
|
|
|
|
|
#include <linux/kallsyms.h>
|
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
|
#include <linux/kprobes.h>
|
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
|
#include <linux/kdebug.h>
|
2010-05-21 10:04:25 +08:00
|
|
|
#include <linux/kgdb.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/kernel.h>
|
2008-02-26 18:15:50 +08:00
|
|
|
#include <linux/module.h>
|
|
|
|
|
#include <linux/ptrace.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/string.h>
|
2008-02-26 18:15:50 +08:00
|
|
|
#include <linux/delay.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/errno.h>
|
2008-02-26 18:15:50 +08:00
|
|
|
#include <linux/kexec.h>
|
|
|
|
|
#include <linux/sched.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/timer.h>
|
|
|
|
|
#include <linux/init.h>
|
2006-12-08 18:36:21 +08:00
|
|
|
#include <linux/bug.h>
|
2008-02-26 18:15:50 +08:00
|
|
|
#include <linux/nmi.h>
|
|
|
|
|
#include <linux/mm.h>
|
2008-10-04 05:17:11 +08:00
|
|
|
#include <linux/smp.h>
|
|
|
|
|
#include <linux/io.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_EISA
|
|
|
|
|
#include <linux/ioport.h>
|
|
|
|
|
#include <linux/eisa.h>
|
|
|
|
|
#endif
|
|
|
|
|
|
2007-07-19 16:49:46 +08:00
|
|
|
#if defined(CONFIG_EDAC)
|
|
|
|
|
#include <linux/edac.h>
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-04-04 06:53:23 +08:00
|
|
|
#include <asm/kmemcheck.h>
|
2008-02-26 18:15:50 +08:00
|
|
|
#include <asm/stacktrace.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/processor.h>
|
|
|
|
|
#include <asm/debugreg.h>
|
2011-07-27 07:09:06 +08:00
|
|
|
#include <linux/atomic.h>
|
2011-08-16 21:57:10 +08:00
|
|
|
#include <asm/ftrace.h>
|
2008-10-04 05:17:11 +08:00
|
|
|
#include <asm/traps.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/desc.h>
|
|
|
|
|
#include <asm/i387.h>
|
2012-02-22 05:19:22 +08:00
|
|
|
#include <asm/fpu-internal.h>
|
2009-06-15 16:22:15 +08:00
|
|
|
#include <asm/mce.h>
|
2008-10-04 05:17:11 +08:00
|
|
|
|
2009-01-29 02:34:09 +08:00
|
|
|
#include <asm/mach_traps.h>
|
2008-10-04 05:17:11 +08:00
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_64
|
2009-08-20 16:35:46 +08:00
|
|
|
#include <asm/x86_init.h>
|
2008-10-04 04:00:39 +08:00
|
|
|
#include <asm/pgalloc.h>
|
|
|
|
|
#include <asm/proto.h>
|
|
|
|
|
#else
|
2008-10-04 05:17:11 +08:00
|
|
|
#include <asm/processor-flags.h>
|
2009-02-23 07:34:39 +08:00
|
|
|
#include <asm/setup.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
asmlinkage int system_call(void);
|
|
|
|
|
|
|
|
|
|
/* Do we ignore FPU interrupts ? */
|
2008-02-26 18:15:50 +08:00
|
|
|
char ignore_fpu_irq;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The IDT has to be page-aligned to simplify the Pentium
|
2009-09-17 04:44:29 +08:00
|
|
|
* F0 0F bug workaround.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2009-09-17 04:44:29 +08:00
|
|
|
gate_desc idt_table[NR_VECTORS] __page_aligned_data = { { { { 0, 0 } } }, };
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-12-20 07:23:44 +08:00
|
|
|
DECLARE_BITMAP(used_vectors, NR_VECTORS);
|
|
|
|
|
EXPORT_SYMBOL_GPL(used_vectors);
|
|
|
|
|
|
2008-09-10 03:55:55 +08:00
|
|
|
static inline void conditional_sti(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
if (regs->flags & X86_EFLAGS_IF)
|
|
|
|
|
local_irq_enable();
|
|
|
|
|
}
|
|
|
|
|
|
2008-10-01 00:41:37 +08:00
|
|
|
static inline void preempt_conditional_sti(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
inc_preempt_count();
|
|
|
|
|
if (regs->flags & X86_EFLAGS_IF)
|
|
|
|
|
local_irq_enable();
|
|
|
|
|
}
|
|
|
|
|
|
2009-01-14 06:36:34 +08:00
|
|
|
static inline void conditional_cli(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
if (regs->flags & X86_EFLAGS_IF)
|
|
|
|
|
local_irq_disable();
|
|
|
|
|
}
|
|
|
|
|
|
2008-10-01 00:41:37 +08:00
|
|
|
static inline void preempt_conditional_cli(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
if (regs->flags & X86_EFLAGS_IF)
|
|
|
|
|
local_irq_disable();
|
|
|
|
|
dec_preempt_count();
|
|
|
|
|
}
|
|
|
|
|
|
2008-02-26 18:15:50 +08:00
|
|
|
static void __kprobes
|
2008-09-26 20:03:08 +08:00
|
|
|
do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
|
2008-02-26 18:15:50 +08:00
|
|
|
long error_code, siginfo_t *info)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2005-06-26 05:58:27 +08:00
|
|
|
struct task_struct *tsk = current;
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-03-28 22:56:57 +08:00
|
|
|
if (regs->flags & X86_VM_MASK) {
|
2008-09-26 20:03:08 +08:00
|
|
|
/*
|
|
|
|
|
* traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
|
|
|
|
|
* On nmi (interrupt 2), do_trap should not be called.
|
|
|
|
|
*/
|
2012-03-10 08:07:10 +08:00
|
|
|
if (trapnr < X86_TRAP_UD)
|
2005-04-17 06:20:36 +08:00
|
|
|
goto vm86_trap;
|
|
|
|
|
goto trap_signal;
|
|
|
|
|
}
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-06-23 15:08:45 +08:00
|
|
|
if (!user_mode(regs))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto kernel_trap;
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-02-26 18:15:50 +08:00
|
|
|
trap_signal:
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2008-02-26 18:15:50 +08:00
|
|
|
/*
|
2012-03-12 17:25:55 +08:00
|
|
|
* We want error_code and trap_nr set for userspace faults and
|
2008-02-26 18:15:50 +08:00
|
|
|
* kernelspace faults which result in die(), but not
|
|
|
|
|
* kernelspace faults which are fixed up. die() gives the
|
|
|
|
|
* process no chance to handle the signal and notice the
|
|
|
|
|
* kernel fault information, so that won't result in polluting
|
|
|
|
|
* the information about previously queued, but not yet
|
|
|
|
|
* delivered, faults. See also do_general_protection below.
|
|
|
|
|
*/
|
|
|
|
|
tsk->thread.error_code = error_code;
|
2012-03-12 17:25:55 +08:00
|
|
|
tsk->thread.trap_nr = trapnr;
|
2007-05-03 01:27:05 +08:00
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
|
|
|
|
|
printk_ratelimit()) {
|
2012-05-22 10:50:07 +08:00
|
|
|
pr_info("%s[%d] trap %s ip:%lx sp:%lx error:%lx",
|
|
|
|
|
tsk->comm, tsk->pid, str,
|
|
|
|
|
regs->ip, regs->sp, error_code);
|
2008-10-04 04:00:39 +08:00
|
|
|
print_vma_addr(" in ", regs->ip);
|
2012-05-22 10:50:07 +08:00
|
|
|
pr_cont("\n");
|
2008-10-04 04:00:39 +08:00
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-02-26 18:15:50 +08:00
|
|
|
if (info)
|
|
|
|
|
force_sig_info(signr, info, tsk);
|
|
|
|
|
else
|
|
|
|
|
force_sig(signr, tsk);
|
|
|
|
|
return;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-02-26 18:15:50 +08:00
|
|
|
kernel_trap:
|
|
|
|
|
if (!fixup_exception(regs)) {
|
|
|
|
|
tsk->thread.error_code = error_code;
|
2012-03-12 17:25:55 +08:00
|
|
|
tsk->thread.trap_nr = trapnr;
|
2008-02-26 18:15:50 +08:00
|
|
|
die(str, regs, error_code);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2008-02-26 18:15:50 +08:00
|
|
|
return;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-02-26 18:15:50 +08:00
|
|
|
vm86_trap:
|
|
|
|
|
if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
|
|
|
|
|
error_code, trapnr))
|
|
|
|
|
goto trap_signal;
|
|
|
|
|
return;
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2008-02-26 18:15:50 +08:00
|
|
|
#define DO_ERROR(trapnr, signr, str, name) \
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
|
2008-02-26 18:15:50 +08:00
|
|
|
{ \
|
|
|
|
|
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
|
2008-07-02 07:29:44 +08:00
|
|
|
== NOTIFY_STOP) \
|
2008-02-26 18:15:50 +08:00
|
|
|
return; \
|
2008-09-10 03:55:56 +08:00
|
|
|
conditional_sti(regs); \
|
2008-09-26 20:03:08 +08:00
|
|
|
do_trap(trapnr, signr, str, regs, error_code, NULL); \
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2008-09-26 20:03:08 +08:00
|
|
|
#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
|
2008-02-26 18:15:50 +08:00
|
|
|
{ \
|
|
|
|
|
siginfo_t info; \
|
|
|
|
|
info.si_signo = signr; \
|
|
|
|
|
info.si_errno = 0; \
|
|
|
|
|
info.si_code = sicode; \
|
|
|
|
|
info.si_addr = (void __user *)siaddr; \
|
|
|
|
|
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
|
2008-07-02 07:29:44 +08:00
|
|
|
== NOTIFY_STOP) \
|
2008-02-26 18:15:50 +08:00
|
|
|
return; \
|
2008-09-10 03:55:56 +08:00
|
|
|
conditional_sti(regs); \
|
2008-09-26 20:03:08 +08:00
|
|
|
do_trap(trapnr, signr, str, regs, error_code, &info); \
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2012-03-10 08:07:10 +08:00
|
|
|
DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV,
|
|
|
|
|
regs->ip)
|
|
|
|
|
DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
|
|
|
|
|
DO_ERROR(X86_TRAP_BR, SIGSEGV, "bounds", bounds)
|
|
|
|
|
DO_ERROR_INFO(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN,
|
|
|
|
|
regs->ip)
|
|
|
|
|
DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",
|
|
|
|
|
coprocessor_segment_overrun)
|
|
|
|
|
DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
|
|
|
|
|
DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2012-03-10 08:07:10 +08:00
|
|
|
DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2012-03-10 08:07:10 +08:00
|
|
|
DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check,
|
|
|
|
|
BUS_ADRALN, 0)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
/* Runs on IST stack */
|
|
|
|
|
dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
|
|
|
|
|
{
|
|
|
|
|
if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
|
2012-03-10 08:07:10 +08:00
|
|
|
X86_TRAP_SS, SIGBUS) == NOTIFY_STOP)
|
2008-10-04 04:00:39 +08:00
|
|
|
return;
|
|
|
|
|
preempt_conditional_sti(regs);
|
2012-03-10 08:07:10 +08:00
|
|
|
do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);
|
2008-10-04 04:00:39 +08:00
|
|
|
preempt_conditional_cli(regs);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
|
|
|
|
|
{
|
|
|
|
|
static const char str[] = "double fault";
|
|
|
|
|
struct task_struct *tsk = current;
|
|
|
|
|
|
|
|
|
|
/* Return not checked because double check cannot be ignored */
|
2012-03-10 08:07:10 +08:00
|
|
|
notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
|
2008-10-04 04:00:39 +08:00
|
|
|
|
|
|
|
|
tsk->thread.error_code = error_code;
|
2012-03-12 17:25:55 +08:00
|
|
|
tsk->thread.trap_nr = X86_TRAP_DF;
|
2008-10-04 04:00:39 +08:00
|
|
|
|
2008-12-26 16:20:22 +08:00
|
|
|
/*
|
|
|
|
|
* This is always a kernel trap and never fixable (and thus must
|
|
|
|
|
* never return).
|
|
|
|
|
*/
|
2008-10-04 04:00:39 +08:00
|
|
|
for (;;)
|
|
|
|
|
die(str, regs, error_code);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void __kprobes
|
2008-07-02 07:32:04 +08:00
|
|
|
do_general_protection(struct pt_regs *regs, long error_code)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-07-02 07:32:04 +08:00
|
|
|
struct task_struct *tsk;
|
2008-02-26 18:15:50 +08:00
|
|
|
|
2008-09-10 03:56:07 +08:00
|
|
|
conditional_sti(regs);
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-03-28 22:56:57 +08:00
|
|
|
if (regs->flags & X86_VM_MASK)
|
2005-04-17 06:20:36 +08:00
|
|
|
goto gp_in_vm86;
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-07-02 07:32:04 +08:00
|
|
|
tsk = current;
|
2005-06-23 15:08:45 +08:00
|
|
|
if (!user_mode(regs))
|
2005-04-17 06:20:36 +08:00
|
|
|
goto gp_in_kernel;
|
|
|
|
|
|
2008-07-02 07:32:04 +08:00
|
|
|
tsk->thread.error_code = error_code;
|
2012-03-12 17:25:55 +08:00
|
|
|
tsk->thread.trap_nr = X86_TRAP_GP;
|
2008-02-26 18:15:50 +08:00
|
|
|
|
2008-07-02 07:32:04 +08:00
|
|
|
if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
|
|
|
|
|
printk_ratelimit()) {
|
2012-05-22 10:50:07 +08:00
|
|
|
pr_info("%s[%d] general protection ip:%lx sp:%lx error:%lx",
|
2008-07-02 07:32:04 +08:00
|
|
|
tsk->comm, task_pid_nr(tsk),
|
|
|
|
|
regs->ip, regs->sp, error_code);
|
2008-01-30 20:33:18 +08:00
|
|
|
print_vma_addr(" in ", regs->ip);
|
2012-05-22 10:50:07 +08:00
|
|
|
pr_cont("\n");
|
2008-01-30 20:33:18 +08:00
|
|
|
}
|
2007-07-22 17:12:28 +08:00
|
|
|
|
2008-07-02 07:32:04 +08:00
|
|
|
force_sig(SIGSEGV, tsk);
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2005-04-17 06:20:36 +08:00
|
|
|
gp_in_vm86:
|
|
|
|
|
local_irq_enable();
|
|
|
|
|
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
|
|
|
|
|
return;
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
gp_in_kernel:
|
2008-07-02 07:32:04 +08:00
|
|
|
if (fixup_exception(regs))
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
tsk->thread.error_code = error_code;
|
2012-03-12 17:25:55 +08:00
|
|
|
tsk->thread.trap_nr = X86_TRAP_GP;
|
2012-03-10 08:07:10 +08:00
|
|
|
if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
|
|
|
|
|
X86_TRAP_GP, SIGSEGV) == NOTIFY_STOP)
|
2008-07-02 07:32:04 +08:00
|
|
|
return;
|
|
|
|
|
die("general protection fault", regs, error_code);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2008-10-04 05:17:11 +08:00
|
|
|
/* May run on IST stack. */
|
2011-08-16 21:57:10 +08:00
|
|
|
dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2011-08-16 21:57:10 +08:00
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
2012-05-31 01:26:37 +08:00
|
|
|
/*
|
|
|
|
|
* ftrace must be first, everything else may cause a recursive crash.
|
|
|
|
|
* See note by declaration of modifying_ftrace_code in ftrace.c
|
|
|
|
|
*/
|
|
|
|
|
if (unlikely(atomic_read(&modifying_ftrace_code)) &&
|
|
|
|
|
ftrace_int3_handler(regs))
|
2011-08-16 21:57:10 +08:00
|
|
|
return;
|
|
|
|
|
#endif
|
2010-05-21 10:04:25 +08:00
|
|
|
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
|
2012-03-10 08:07:10 +08:00
|
|
|
if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
|
|
|
|
|
SIGTRAP) == NOTIFY_STOP)
|
2010-05-21 10:04:25 +08:00
|
|
|
return;
|
|
|
|
|
#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
|
2011-10-25 22:21:59 +08:00
|
|
|
|
2012-03-10 08:07:10 +08:00
|
|
|
if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
|
|
|
|
|
SIGTRAP) == NOTIFY_STOP)
|
2005-05-01 23:58:49 +08:00
|
|
|
return;
|
2008-02-26 18:15:50 +08:00
|
|
|
|
2011-12-17 00:43:02 +08:00
|
|
|
/*
|
|
|
|
|
* Let others (NMI) know that the debug stack is in use
|
|
|
|
|
* as we may switch to the interrupt stack.
|
|
|
|
|
*/
|
|
|
|
|
debug_stack_usage_inc();
|
2008-10-04 04:00:34 +08:00
|
|
|
preempt_conditional_sti(regs);
|
2012-03-10 08:07:10 +08:00
|
|
|
do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
|
2008-10-04 04:00:34 +08:00
|
|
|
preempt_conditional_cli(regs);
|
2011-12-17 00:43:02 +08:00
|
|
|
debug_stack_usage_dec();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_64
|
2008-12-26 16:20:22 +08:00
|
|
|
/*
|
|
|
|
|
* Help handler running on IST stack to switch back to user stack
|
|
|
|
|
* for scheduling or signal handling. The actual stack switch is done in
|
|
|
|
|
* entry.S
|
|
|
|
|
*/
|
2008-10-04 04:00:39 +08:00
|
|
|
asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
|
|
|
|
|
{
|
|
|
|
|
struct pt_regs *regs = eregs;
|
|
|
|
|
/* Did already sync */
|
|
|
|
|
if (eregs == (struct pt_regs *)eregs->sp)
|
|
|
|
|
;
|
|
|
|
|
/* Exception from user space */
|
|
|
|
|
else if (user_mode(eregs))
|
|
|
|
|
regs = task_pt_regs(current);
|
2008-12-26 16:20:22 +08:00
|
|
|
/*
|
|
|
|
|
* Exception from kernel and interrupts are enabled. Move to
|
|
|
|
|
* kernel process stack.
|
|
|
|
|
*/
|
2008-10-04 04:00:39 +08:00
|
|
|
else if (eregs->flags & X86_EFLAGS_IF)
|
|
|
|
|
regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs));
|
|
|
|
|
if (eregs != regs)
|
|
|
|
|
*regs = *eregs;
|
|
|
|
|
return regs;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Our handling of the processor debug registers is non-trivial.
|
|
|
|
|
* We do not clear them on entry and exit from the kernel. Therefore
|
|
|
|
|
* it is possible to get a watchpoint trap here from inside the kernel.
|
|
|
|
|
* However, the code in ./ptrace.c has ensured that the user can
|
|
|
|
|
* only set watchpoints on userspace addresses. Therefore the in-kernel
|
|
|
|
|
* watchpoint trap can only occur in code which is reading/writing
|
|
|
|
|
* from user space. Such code must not hold kernel locks (since it
|
|
|
|
|
* can equally take a page fault), therefore it is safe to call
|
|
|
|
|
* force_sig_info even though that claims and releases locks.
|
2008-02-26 18:15:50 +08:00
|
|
|
*
|
2005-04-17 06:20:36 +08:00
|
|
|
* Code in ./signal.c ensures that the debug control register
|
|
|
|
|
* is restored before we deliver any signal, and therefore that
|
|
|
|
|
* user code runs with the correct debug control register even though
|
|
|
|
|
* we clear it here.
|
|
|
|
|
*
|
|
|
|
|
* Being careful here means that we don't have to be as careful in a
|
|
|
|
|
* lot of more complicated places (task switching can be a bit lazy
|
|
|
|
|
* about restoring all the debug state, and ptrace doesn't have to
|
|
|
|
|
* find every occurrence of the TF bit that could be saved away even
|
|
|
|
|
* by user code)
|
2008-10-04 05:17:11 +08:00
|
|
|
*
|
|
|
|
|
* May run on IST stack.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
|
struct task_struct *tsk = current;
|
2010-06-30 21:09:06 +08:00
|
|
|
int user_icebp = 0;
|
2009-06-02 02:14:08 +08:00
|
|
|
unsigned long dr6;
|
2008-09-23 17:53:52 +08:00
|
|
|
int si_code;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-06-02 02:14:08 +08:00
|
|
|
get_debugreg(dr6, 6);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2010-01-28 19:14:01 +08:00
|
|
|
/* Filter out all the reserved bits which are preset to 1 */
|
|
|
|
|
dr6 &= ~DR6_RESERVED;
|
|
|
|
|
|
2010-06-30 21:09:06 +08:00
|
|
|
/*
|
|
|
|
|
* If dr6 has no reason to give us about the origin of this trap,
|
|
|
|
|
* then it's very likely the result of an icebp/int01 trap.
|
|
|
|
|
* User wants a sigtrap for that.
|
|
|
|
|
*/
|
|
|
|
|
if (!dr6 && user_mode(regs))
|
|
|
|
|
user_icebp = 1;
|
|
|
|
|
|
2008-04-04 06:53:23 +08:00
|
|
|
/* Catch kmemcheck conditions first of all! */
|
2009-06-17 18:52:15 +08:00
|
|
|
if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
|
2008-04-04 06:53:23 +08:00
|
|
|
return;
|
|
|
|
|
|
2009-06-02 02:14:08 +08:00
|
|
|
/* DR6 may or may not be cleared by the CPU */
|
|
|
|
|
set_debugreg(0, 6);
|
2008-01-30 20:30:54 +08:00
|
|
|
|
2010-03-25 21:51:51 +08:00
|
|
|
/*
|
|
|
|
|
* The processor cleared BTF, so don't mark that we need it set.
|
|
|
|
|
*/
|
|
|
|
|
clear_tsk_thread_flag(tsk, TIF_BLOCKSTEP);
|
|
|
|
|
|
2009-06-02 02:14:08 +08:00
|
|
|
/* Store the virtualized DR6 value */
|
|
|
|
|
tsk->thread.debugreg6 = dr6;
|
|
|
|
|
|
2009-06-02 02:17:06 +08:00
|
|
|
if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code,
|
|
|
|
|
SIGTRAP) == NOTIFY_STOP)
|
2005-04-17 06:20:36 +08:00
|
|
|
return;
|
2008-10-01 00:41:37 +08:00
|
|
|
|
2011-12-17 00:43:02 +08:00
|
|
|
/*
|
|
|
|
|
* Let others (NMI) know that the debug stack is in use
|
|
|
|
|
* as we may switch to the interrupt stack.
|
|
|
|
|
*/
|
|
|
|
|
debug_stack_usage_inc();
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* It's safe to allow irq's after DR6 has been saved */
|
2008-10-01 00:41:37 +08:00
|
|
|
preempt_conditional_sti(regs);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-06-02 02:14:08 +08:00
|
|
|
if (regs->flags & X86_VM_MASK) {
|
2012-03-10 08:07:10 +08:00
|
|
|
handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
|
|
|
|
|
X86_TRAP_DB);
|
x86, vm86: Fix preemption bug for int1 debug and int3 breakpoint handlers.
Impact: fix kernel bug such as:
BUG: scheduling while atomic: dosemu.bin/19680/0x00000004
See also Ubuntu bug 455067 at
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/455067
Commits 4915a35e35a037254550a2ba9f367a812bc37d40
("Use preempt_conditional_sti/cli in do_int3, like on x86_64.")
and 3d2a71a596bd9c761c8487a2178e95f8a61da083
("x86, traps: converge do_debug handlers")
started disabling preemption in int1 and int3 handlers on i386.
The problem with vm86 is that the call to handle_vm86_trap() may jump
straight to entry_32.S and never returns so preempt is never enabled
again, and there is an imbalance in the preempt count.
Commit be716615fe596ee117292dc615e95f707fb67fd1 ("x86, vm86:
fix preemption bug"), which was later (accidentally?) reverted by commit
08d68323d1f0c34452e614263b212ca556dae47f ("hw-breakpoints: modifying
generic debug exception to use thread-specific debug registers")
fixed the problem for debug exceptions but not for breakpoints.
There are three solutions to this problem.
1. Reenable preemption before calling handle_vm86_trap(). This
was the approach that was later reverted.
2. Do not disable preemption for i386 in breakpoint and debug handlers.
This was the situation before October 2008. As far as I understand
preemption only needs to be disabled on x86_64 because a seperate stack is
used, but it's nice to have things work the same way on
i386 and x86_64.
3. Let handle_vm86_trap() return instead of jumping to assembly code.
By setting a flag in _TIF_WORK_MASK, either TIF_IRET or TIF_NOTIFY_RESUME,
the code in entry_32.S is instructed to return to 32 bit mode from
V86 mode. The logic in entry_32.S was already present to handle signals.
(I chose TIF_IRET because it's slightly more efficient in
do_notify_resume() in signal.c, but in fact TIF_IRET can probably be
replaced by TIF_NOTIFY_RESUME everywhere.)
I'm submitting approach 3, because I believe it is the most elegant
and prevents future confusion. Still, an obvious
preempt_conditional_cli(regs); is necessary in traps.c to correct the
bug.
[ hpa: This is technically a regression, but because:
1. the regression is so old,
2. the patch seems relatively high risk, justifying more testing, and
3. we're late in the 2.6.36-rc cycle,
I'm queuing it up for the 2.6.37 merge window. It might, however,
justify as a -stable backport at a latter time, hence Cc: stable. ]
Signed-off-by: Bart Oldeman <bartoldeman@users.sourceforge.net>
LKML-Reference: <alpine.DEB.2.00.1009231312330.4732@localhost.localdomain>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: K.Prasad <prasad@linux.vnet.ibm.com>
Cc: Alan Stern <stern@rowland.harvard.edu>
Cc: Alexander van Heukelum <heukelum@fastmail.fm>
Cc: <stable@kernel.org>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2010-09-24 01:16:58 +08:00
|
|
|
preempt_conditional_cli(regs);
|
2011-12-17 00:43:02 +08:00
|
|
|
debug_stack_usage_dec();
|
2009-06-02 02:14:08 +08:00
|
|
|
return;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
2009-06-02 02:14:08 +08:00
|
|
|
* Single-stepping through system calls: ignore any exceptions in
|
|
|
|
|
* kernel space, but re-enable TF when returning to user mode.
|
|
|
|
|
*
|
|
|
|
|
* We already checked v86 mode above, so we can check for kernel mode
|
|
|
|
|
* by just checking the CPL of CS.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2009-06-02 02:14:08 +08:00
|
|
|
if ((dr6 & DR_STEP) && !user_mode(regs)) {
|
|
|
|
|
tsk->thread.debugreg6 &= ~DR_STEP;
|
|
|
|
|
set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
|
|
|
|
|
regs->flags &= ~X86_EFLAGS_TF;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2009-06-02 02:14:08 +08:00
|
|
|
si_code = get_si_code(tsk->thread.debugreg6);
|
2010-06-30 21:09:06 +08:00
|
|
|
if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
|
2009-06-02 02:14:08 +08:00
|
|
|
send_sigtrap(tsk, regs, error_code, si_code);
|
2008-10-01 00:41:37 +08:00
|
|
|
preempt_conditional_cli(regs);
|
2011-12-17 00:43:02 +08:00
|
|
|
debug_stack_usage_dec();
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Note that we play around with the 'TS' bit in an attempt to get
|
|
|
|
|
* the correct behaviour even in the presence of the asynchronous
|
|
|
|
|
* IRQ13 behaviour
|
|
|
|
|
*/
|
2010-03-21 21:00:44 +08:00
|
|
|
void math_error(struct pt_regs *regs, int error_code, int trapnr)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2010-03-21 21:00:45 +08:00
|
|
|
struct task_struct *task = current;
|
2005-04-17 06:20:36 +08:00
|
|
|
siginfo_t info;
|
2010-03-21 21:00:44 +08:00
|
|
|
unsigned short err;
|
2012-03-10 08:07:10 +08:00
|
|
|
char *str = (trapnr == X86_TRAP_MF) ? "fpu exception" :
|
|
|
|
|
"simd exception";
|
2010-03-21 21:00:45 +08:00
|
|
|
|
|
|
|
|
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
|
|
|
|
|
return;
|
|
|
|
|
conditional_sti(regs);
|
|
|
|
|
|
|
|
|
|
if (!user_mode_vm(regs))
|
|
|
|
|
{
|
|
|
|
|
if (!fixup_exception(regs)) {
|
|
|
|
|
task->thread.error_code = error_code;
|
2012-03-12 17:25:55 +08:00
|
|
|
task->thread.trap_nr = trapnr;
|
2010-03-21 21:00:45 +08:00
|
|
|
die(str, regs, error_code);
|
|
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Save the info for the exception handler and clear the error.
|
|
|
|
|
*/
|
|
|
|
|
save_init_fpu(task);
|
2012-03-12 17:25:55 +08:00
|
|
|
task->thread.trap_nr = trapnr;
|
2010-03-21 21:00:44 +08:00
|
|
|
task->thread.error_code = error_code;
|
2005-04-17 06:20:36 +08:00
|
|
|
info.si_signo = SIGFPE;
|
|
|
|
|
info.si_errno = 0;
|
2010-03-21 21:00:44 +08:00
|
|
|
info.si_addr = (void __user *)regs->ip;
|
2012-03-10 08:07:10 +08:00
|
|
|
if (trapnr == X86_TRAP_MF) {
|
2010-03-21 21:00:44 +08:00
|
|
|
unsigned short cwd, swd;
|
|
|
|
|
/*
|
|
|
|
|
* (~cwd & swd) will mask out exceptions that are not set to unmasked
|
|
|
|
|
* status. 0x3f is the exception bits in these regs, 0x200 is the
|
|
|
|
|
* C1 reg you need in case of a stack fault, 0x040 is the stack
|
|
|
|
|
* fault bit. We should only be taking one exception at a time,
|
|
|
|
|
* so if this combination doesn't produce any single exception,
|
|
|
|
|
* then we have a bad program that isn't synchronizing its FPU usage
|
|
|
|
|
* and it will suffer the consequences since we won't be able to
|
|
|
|
|
* fully reproduce the context of the exception
|
|
|
|
|
*/
|
|
|
|
|
cwd = get_fpu_cwd(task);
|
|
|
|
|
swd = get_fpu_swd(task);
|
2008-12-23 09:56:05 +08:00
|
|
|
|
2010-03-21 21:00:44 +08:00
|
|
|
err = swd & ~cwd;
|
|
|
|
|
} else {
|
|
|
|
|
/*
|
|
|
|
|
* The SIMD FPU exceptions are handled a little differently, as there
|
|
|
|
|
* is only a single status/control register. Thus, to determine which
|
|
|
|
|
* unmasked exception was caught we must mask the exception mask bits
|
|
|
|
|
* at 0x1f80, and then use these to mask the exception bits at 0x3f.
|
|
|
|
|
*/
|
|
|
|
|
unsigned short mxcsr = get_fpu_mxcsr(task);
|
|
|
|
|
err = ~(mxcsr >> 7) & mxcsr;
|
|
|
|
|
}
|
2008-12-23 09:56:05 +08:00
|
|
|
|
|
|
|
|
if (err & 0x001) { /* Invalid op */
|
2008-02-26 18:15:50 +08:00
|
|
|
/*
|
|
|
|
|
* swd & 0x240 == 0x040: Stack Underflow
|
|
|
|
|
* swd & 0x240 == 0x240: Stack Overflow
|
|
|
|
|
* User must clear the SF bit (0x40) if set
|
|
|
|
|
*/
|
|
|
|
|
info.si_code = FPE_FLTINV;
|
2008-12-23 09:56:05 +08:00
|
|
|
} else if (err & 0x004) { /* Divide by Zero */
|
2008-02-26 18:15:50 +08:00
|
|
|
info.si_code = FPE_FLTDIV;
|
2008-12-23 09:56:05 +08:00
|
|
|
} else if (err & 0x008) { /* Overflow */
|
2008-02-26 18:15:50 +08:00
|
|
|
info.si_code = FPE_FLTOVF;
|
2008-12-23 09:56:05 +08:00
|
|
|
} else if (err & 0x012) { /* Denormal, Underflow */
|
|
|
|
|
info.si_code = FPE_FLTUND;
|
|
|
|
|
} else if (err & 0x020) { /* Precision */
|
2008-02-26 18:15:50 +08:00
|
|
|
info.si_code = FPE_FLTRES;
|
2008-12-23 09:56:05 +08:00
|
|
|
} else {
|
2008-12-26 16:20:22 +08:00
|
|
|
/*
|
2012-03-10 08:07:10 +08:00
|
|
|
* If we're using IRQ 13, or supposedly even some trap
|
|
|
|
|
* X86_TRAP_MF implementations, it's possible
|
|
|
|
|
* we get a spurious trap, which is not an error.
|
2008-12-26 16:20:22 +08:00
|
|
|
*/
|
2012-03-10 08:07:10 +08:00
|
|
|
return;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
force_sig_info(SIGFPE, &info, task);
|
|
|
|
|
}
|
|
|
|
|
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2005-04-17 06:20:36 +08:00
|
|
|
ignore_fpu_irq = 1;
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
|
|
|
|
|
2012-03-10 08:07:10 +08:00
|
|
|
math_error(regs, error_code, X86_TRAP_MF);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void
|
|
|
|
|
do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2012-03-10 08:07:10 +08:00
|
|
|
math_error(regs, error_code, X86_TRAP_XF);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void
|
|
|
|
|
do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-09-10 03:56:08 +08:00
|
|
|
conditional_sti(regs);
|
2005-04-17 06:20:36 +08:00
|
|
|
#if 0
|
|
|
|
|
/* No need to warn about this any longer. */
|
2012-05-22 10:50:07 +08:00
|
|
|
pr_info("Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
|
}
|
x86, mce: use 64bit machine check code on 32bit
The 64bit machine check code is in many ways much better than
the 32bit machine check code: it is more specification compliant,
is cleaner, only has a single code base versus one per CPU,
has better infrastructure for recovery, has a cleaner way to communicate
with user space etc. etc.
Use the 64bit code for 32bit too.
This is the second attempt to do this. There was one a couple of years
ago to unify this code for 32bit and 64bit. Back then this ran into some
trouble with K7s and was reverted.
I believe this time the K7 problems (and some others) are addressed.
I went over the old handlers and was very careful to retain
all quirks.
But of course this needs a lot of testing on old systems. On newer
64bit capable systems I don't expect much problems because they have been
already tested with the 64bit kernel.
I made this a CONFIG for now that still allows to select the old
machine check code. This is mostly to make testing easier,
if someone runs into a problem we can ask them to try
with the CONFIG switched.
The new code is default y for more coverage.
Once there is confidence the 64bit code works well on older hardware
too the CONFIG_X86_OLD_MCE and the associated code can be easily
removed.
This causes a behaviour change for 32bit installations. They now
have to install the mcelog package to be able to log
corrected machine checks.
The 64bit machine check code only handles CPUs which support the
standard Intel machine check architecture described in the IA32 SDM.
The 32bit code has special support for some older CPUs which
have non standard machine check architectures, in particular
WinChip C3 and Intel P5. I made those a separate CONFIG option
and kept them for now. The WinChip variant could be probably
removed without too much pain, it doesn't really do anything
interesting. P5 is also disabled by default (like it
was before) because many motherboards have it miswired, but
according to Alan Cox a few embedded setups use that one.
Forward ported/heavily changed version of old patch, original patch
included review/fixes from Thomas Gleixner, Bert Wesarg.
Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
2009-04-29 01:07:31 +08:00
|
|
|
|
2009-04-29 05:32:56 +08:00
|
|
|
asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void)
|
2008-10-04 04:00:39 +08:00
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2008-02-26 18:15:50 +08:00
|
|
|
* 'math_state_restore()' saves the current math information in the
|
2005-04-17 06:20:36 +08:00
|
|
|
* old math state array, and gets the new ones from the current task
|
|
|
|
|
*
|
|
|
|
|
* Careful.. There are problems with IBM-designed IRQ13 behaviour.
|
|
|
|
|
* Don't touch unless you *really* know how it works.
|
|
|
|
|
*
|
2012-02-14 05:47:25 +08:00
|
|
|
* Must be called with kernel preemption disabled (eg with local
|
|
|
|
|
* local interrupts as in the case of do_device_not_available).
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2012-02-14 05:47:25 +08:00
|
|
|
void math_state_restore(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
i387: move TS_USEDFPU flag from thread_info to task_struct
This moves the bit that indicates whether a thread has ownership of the
FPU from the TS_USEDFPU bit in thread_info->status to a word of its own
(called 'has_fpu') in task_struct->thread.has_fpu.
This fixes two independent bugs at the same time:
- changing 'thread_info->status' from the scheduler causes nasty
problems for the other users of that variable, since it is defined to
be thread-synchronous (that's what the "TS_" part of the naming was
supposed to indicate).
So perfectly valid code could (and did) do
ti->status |= TS_RESTORE_SIGMASK;
and the compiler was free to do that as separate load, or and store
instructions. Which can cause problems with preemption, since a task
switch could happen in between, and change the TS_USEDFPU bit. The
change to TS_USEDFPU would be overwritten by the final store.
In practice, this seldom happened, though, because the 'status' field
was seldom used more than once, so gcc would generally tend to
generate code that used a read-modify-write instruction and thus
happened to avoid this problem - RMW instructions are naturally low
fat and preemption-safe.
- On x86-32, the current_thread_info() pointer would, during interrupts
and softirqs, point to a *copy* of the real thread_info, because
x86-32 uses %esp to calculate the thread_info address, and thus the
separate irq (and softirq) stacks would cause these kinds of odd
thread_info copy aliases.
This is normally not a problem, since interrupts aren't supposed to
look at thread information anyway (what thread is running at
interrupt time really isn't very well-defined), but it confused the
heck out of irq_fpu_usable() and the code that tried to squirrel
away the FPU state.
(It also caused untold confusion for us poor kernel developers).
It also turns out that using 'task_struct' is actually much more natural
for most of the call sites that care about the FPU state, since they
tend to work with the task struct for other reasons anyway (ie
scheduling). And the FPU data that we are going to save/restore is
found there too.
Thanks to Arjan Van De Ven <arjan@linux.intel.com> for pointing us to
the %esp issue.
Cc: Arjan van de Ven <arjan@linux.intel.com>
Reported-and-tested-by: Raphael Prevost <raphael@buro.asia>
Acked-and-tested-by: Suresh Siddha <suresh.b.siddha@intel.com>
Tested-by: Peter Anvin <hpa@zytor.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-02-18 13:48:54 +08:00
|
|
|
struct task_struct *tsk = current;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-03-11 06:28:05 +08:00
|
|
|
if (!tsk_used_math(tsk)) {
|
|
|
|
|
local_irq_enable();
|
|
|
|
|
/*
|
|
|
|
|
* does a slab alloc which can sleep
|
|
|
|
|
*/
|
|
|
|
|
if (init_fpu(tsk)) {
|
|
|
|
|
/*
|
|
|
|
|
* ran out of memory!
|
|
|
|
|
*/
|
|
|
|
|
do_group_exit(SIGKILL);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
local_irq_disable();
|
|
|
|
|
}
|
|
|
|
|
|
i387: move TS_USEDFPU flag from thread_info to task_struct
This moves the bit that indicates whether a thread has ownership of the
FPU from the TS_USEDFPU bit in thread_info->status to a word of its own
(called 'has_fpu') in task_struct->thread.has_fpu.
This fixes two independent bugs at the same time:
- changing 'thread_info->status' from the scheduler causes nasty
problems for the other users of that variable, since it is defined to
be thread-synchronous (that's what the "TS_" part of the naming was
supposed to indicate).
So perfectly valid code could (and did) do
ti->status |= TS_RESTORE_SIGMASK;
and the compiler was free to do that as separate load, or and store
instructions. Which can cause problems with preemption, since a task
switch could happen in between, and change the TS_USEDFPU bit. The
change to TS_USEDFPU would be overwritten by the final store.
In practice, this seldom happened, though, because the 'status' field
was seldom used more than once, so gcc would generally tend to
generate code that used a read-modify-write instruction and thus
happened to avoid this problem - RMW instructions are naturally low
fat and preemption-safe.
- On x86-32, the current_thread_info() pointer would, during interrupts
and softirqs, point to a *copy* of the real thread_info, because
x86-32 uses %esp to calculate the thread_info address, and thus the
separate irq (and softirq) stacks would cause these kinds of odd
thread_info copy aliases.
This is normally not a problem, since interrupts aren't supposed to
look at thread information anyway (what thread is running at
interrupt time really isn't very well-defined), but it confused the
heck out of irq_fpu_usable() and the code that tried to squirrel
away the FPU state.
(It also caused untold confusion for us poor kernel developers).
It also turns out that using 'task_struct' is actually much more natural
for most of the call sites that care about the FPU state, since they
tend to work with the task struct for other reasons anyway (ie
scheduling). And the FPU data that we are going to save/restore is
found there too.
Thanks to Arjan Van De Ven <arjan@linux.intel.com> for pointing us to
the %esp issue.
Cc: Arjan van de Ven <arjan@linux.intel.com>
Reported-and-tested-by: Raphael Prevost <raphael@buro.asia>
Acked-and-tested-by: Suresh Siddha <suresh.b.siddha@intel.com>
Tested-by: Peter Anvin <hpa@zytor.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-02-18 13:48:54 +08:00
|
|
|
__thread_fpu_begin(tsk);
|
2012-02-20 03:48:44 +08:00
|
|
|
/*
|
|
|
|
|
* Paranoid restore. send a SIGSEGV if we fail to restore the state.
|
|
|
|
|
*/
|
|
|
|
|
if (unlikely(restore_fpu_checking(tsk))) {
|
|
|
|
|
__thread_fpu_end(tsk);
|
|
|
|
|
force_sig(SIGSEGV, tsk);
|
|
|
|
|
return;
|
|
|
|
|
}
|
i387: do not preload FPU state at task switch time
Yes, taking the trap to re-load the FPU/MMX state is expensive, but so
is spending several days looking for a bug in the state save/restore
code. And the preload code has some rather subtle interactions with
both paravirtualization support and segment state restore, so it's not
nearly as simple as it should be.
Also, now that we no longer necessarily depend on a single bit (ie
TS_USEDFPU) for keeping track of the state of the FPU, we migth be able
to do better. If we are really switching between two processes that
keep touching the FP state, save/restore is inevitable, but in the case
of having one process that does most of the FPU usage, we may actually
be able to do much better than the preloading.
In particular, we may be able to keep track of which CPU the process ran
on last, and also per CPU keep track of which process' FP state that CPU
has. For modern CPU's that don't destroy the FPU contents on save time,
that would allow us to do a lazy restore by just re-enabling the
existing FPU state - with no restore cost at all!
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-02-17 07:45:23 +08:00
|
|
|
|
|
|
|
|
tsk->fpu_counter++;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2007-07-19 16:49:21 +08:00
|
|
|
EXPORT_SYMBOL_GPL(math_state_restore);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void __kprobes
|
2009-02-10 22:51:45 +08:00
|
|
|
do_device_not_available(struct pt_regs *regs, long error_code)
|
2008-09-10 03:56:02 +08:00
|
|
|
{
|
2010-09-04 09:17:15 +08:00
|
|
|
#ifdef CONFIG_MATH_EMULATION
|
2008-09-10 03:56:02 +08:00
|
|
|
if (read_cr0() & X86_CR0_EM) {
|
2009-02-09 21:17:39 +08:00
|
|
|
struct math_emu_info info = { };
|
|
|
|
|
|
2008-09-10 03:56:02 +08:00
|
|
|
conditional_sti(regs);
|
2009-02-09 21:17:39 +08:00
|
|
|
|
2009-02-10 22:51:45 +08:00
|
|
|
info.regs = regs;
|
2009-02-09 21:17:39 +08:00
|
|
|
math_emulate(&info);
|
2010-09-04 09:17:15 +08:00
|
|
|
return;
|
2008-09-10 03:56:02 +08:00
|
|
|
}
|
2010-09-04 09:17:15 +08:00
|
|
|
#endif
|
|
|
|
|
math_state_restore(); /* interrupts still off */
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
|
conditional_sti(regs);
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2008-09-10 03:56:02 +08:00
|
|
|
}
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-10-01 00:41:36 +08:00
|
|
|
dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
|
2008-09-10 03:56:13 +08:00
|
|
|
{
|
|
|
|
|
siginfo_t info;
|
|
|
|
|
local_irq_enable();
|
|
|
|
|
|
|
|
|
|
info.si_signo = SIGILL;
|
|
|
|
|
info.si_errno = 0;
|
|
|
|
|
info.si_code = ILL_BADSTK;
|
2009-02-22 08:00:57 +08:00
|
|
|
info.si_addr = NULL;
|
2012-03-10 08:07:10 +08:00
|
|
|
if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
|
|
|
|
|
X86_TRAP_IRET, SIGILL) == NOTIFY_STOP)
|
2008-09-10 03:56:13 +08:00
|
|
|
return;
|
2012-03-10 08:07:10 +08:00
|
|
|
do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
|
|
|
|
|
&info);
|
2008-09-10 03:56:13 +08:00
|
|
|
}
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2008-09-10 03:56:13 +08:00
|
|
|
|
2010-05-21 10:04:29 +08:00
|
|
|
/* Set of traps needed for early debugging. */
|
|
|
|
|
void __init early_trap_init(void)
|
|
|
|
|
{
|
2012-03-10 08:07:10 +08:00
|
|
|
set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
|
2010-05-21 10:04:29 +08:00
|
|
|
/* int3 can be called from all */
|
2012-03-10 08:07:10 +08:00
|
|
|
set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
|
|
|
|
|
set_intr_gate(X86_TRAP_PF, &page_fault);
|
2010-05-21 10:04:29 +08:00
|
|
|
load_idt(&idt_descr);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
void __init trap_init(void)
|
|
|
|
|
{
|
2007-10-20 02:35:03 +08:00
|
|
|
int i;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_EISA
|
2008-01-30 20:33:49 +08:00
|
|
|
void __iomem *p = early_ioremap(0x0FFFD9, 4);
|
2008-02-26 18:15:50 +08:00
|
|
|
|
|
|
|
|
if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
|
2005-04-17 06:20:36 +08:00
|
|
|
EISA_bus = 1;
|
2008-01-30 20:33:49 +08:00
|
|
|
early_iounmap(p, 4);
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
|
2012-03-10 08:07:10 +08:00
|
|
|
set_intr_gate(X86_TRAP_DE, ÷_error);
|
|
|
|
|
set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
|
2008-10-04 04:00:32 +08:00
|
|
|
/* int4 can be called from all */
|
2012-03-10 08:07:10 +08:00
|
|
|
set_system_intr_gate(X86_TRAP_OF, &overflow);
|
|
|
|
|
set_intr_gate(X86_TRAP_BR, &bounds);
|
|
|
|
|
set_intr_gate(X86_TRAP_UD, &invalid_op);
|
|
|
|
|
set_intr_gate(X86_TRAP_NM, &device_not_available);
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_X86_32
|
2012-03-10 08:07:10 +08:00
|
|
|
set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
|
2008-10-04 04:00:39 +08:00
|
|
|
#else
|
2012-03-10 08:07:10 +08:00
|
|
|
set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2012-03-10 08:07:10 +08:00
|
|
|
set_intr_gate(X86_TRAP_OLD_MF, &coprocessor_segment_overrun);
|
|
|
|
|
set_intr_gate(X86_TRAP_TS, &invalid_TSS);
|
|
|
|
|
set_intr_gate(X86_TRAP_NP, &segment_not_present);
|
|
|
|
|
set_intr_gate_ist(X86_TRAP_SS, &stack_segment, STACKFAULT_STACK);
|
|
|
|
|
set_intr_gate(X86_TRAP_GP, &general_protection);
|
|
|
|
|
set_intr_gate(X86_TRAP_SPURIOUS, &spurious_interrupt_bug);
|
|
|
|
|
set_intr_gate(X86_TRAP_MF, &coprocessor_error);
|
|
|
|
|
set_intr_gate(X86_TRAP_AC, &alignment_check);
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_X86_MCE
|
2012-03-10 08:07:10 +08:00
|
|
|
set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
2012-03-10 08:07:10 +08:00
|
|
|
set_intr_gate(X86_TRAP_XF, &simd_coprocessor_error);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-01-25 18:38:09 +08:00
|
|
|
/* Reserve all the builtin and the syscall vector: */
|
|
|
|
|
for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
|
|
|
|
|
set_bit(i, used_vectors);
|
|
|
|
|
|
2008-10-04 04:00:39 +08:00
|
|
|
#ifdef CONFIG_IA32_EMULATION
|
|
|
|
|
set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
|
2009-01-25 18:38:09 +08:00
|
|
|
set_bit(IA32_SYSCALL_VECTOR, used_vectors);
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
2008-10-04 04:00:32 +08:00
|
|
|
set_system_trap_gate(SYSCALL_VECTOR, &system_call);
|
2007-10-20 02:35:03 +08:00
|
|
|
set_bit(SYSCALL_VECTOR, used_vectors);
|
2008-10-04 04:00:39 +08:00
|
|
|
#endif
|
2009-01-25 18:38:09 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2008-02-26 18:15:50 +08:00
|
|
|
* Should be a barrier for any external CPU state:
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
|
cpu_init();
|
|
|
|
|
|
2009-08-20 16:35:46 +08:00
|
|
|
x86_init.irqs.trap_init();
|
2011-12-09 16:02:19 +08:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
memcpy(&nmi_idt_table, &idt_table, IDT_ENTRIES * 16);
|
2012-03-10 08:07:10 +08:00
|
|
|
set_nmi_gate(X86_TRAP_DB, &debug);
|
|
|
|
|
set_nmi_gate(X86_TRAP_BP, &int3);
|
2011-12-09 16:02:19 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|