hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[MIPS] FPU ownership management & preemption fixes 

Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
This commit is contained in:
Atsushi Nemoto 2007-03-10 01:07:45 +09:00 committed by Ralf Baechle
parent c6a2f46793
commit 53dc80287d
10 changed files with 126 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
arch/mips/kernel/r2300_switch.S
View File

@ -49,8 +49,7 @@ LEAF(resume)
#ifndef CONFIG_CPU_HAS_LLSC #ifndef CONFIG_CPU_HAS_LLSC
sw zero, ll_bit sw zero, ll_bit
#endif #endif
mfc0 t1, CP0_STATUS mfc0 t2, CP0_STATUS
sw t1, THREAD_STATUS(a0)
cpu_save_nonscratch a0 cpu_save_nonscratch a0
sw ra, THREAD_REG31(a0) sw ra, THREAD_REG31(a0)
@ -60,8 +59,8 @@ LEAF(resume)
lw t3, TASK_THREAD_INFO(a0) lw t3, TASK_THREAD_INFO(a0)
lw t0, TI_FLAGS(t3) lw t0, TI_FLAGS(t3)
li t1, _TIF_USEDFPU li t1, _TIF_USEDFPU
and t2, t0, t1 and t1, t0
beqz t2, 1f beqz t1, 1f
nor t1, zero, t1 nor t1, zero, t1
and t0, t0, t1 and t0, t0, t1
@ -74,10 +73,13 @@ LEAF(resume)
li t1, ~ST0_CU1 li t1, ~ST0_CU1
and t0, t0, t1 and t0, t0, t1
sw t0, ST_OFF(t3) sw t0, ST_OFF(t3)
/* clear thread_struct CU1 bit */
and t2, t1
fpu_save_single a0, t0 # clobbers t0 fpu_save_single a0, t0 # clobbers t0
1: 1:
sw t2, THREAD_STATUS(a0)
/* /*
* The order of restoring the registers takes care of the race * The order of restoring the registers takes care of the race
* updating $28, $29 and kernelsp without disabling ints. * updating $28, $29 and kernelsp without disabling ints.

10
arch/mips/kernel/r4k_switch.S
View File

@ -48,8 +48,7 @@
#ifndef CONFIG_CPU_HAS_LLSC #ifndef CONFIG_CPU_HAS_LLSC
sw zero, ll_bit sw zero, ll_bit
#endif #endif
mfc0 t1, CP0_STATUS mfc0 t2, CP0_STATUS
LONG_S t1, THREAD_STATUS(a0)
cpu_save_nonscratch a0 cpu_save_nonscratch a0
LONG_S ra, THREAD_REG31(a0) LONG_S ra, THREAD_REG31(a0)
@ -59,8 +58,8 @@
PTR_L t3, TASK_THREAD_INFO(a0) PTR_L t3, TASK_THREAD_INFO(a0)
LONG_L t0, TI_FLAGS(t3) LONG_L t0, TI_FLAGS(t3)
li t1, _TIF_USEDFPU li t1, _TIF_USEDFPU
and t2, t0, t1 and t1, t0
beqz t2, 1f beqz t1, 1f
nor t1, zero, t1 nor t1, zero, t1
and t0, t0, t1 and t0, t0, t1
@ -73,10 +72,13 @@
li t1, ~ST0_CU1 li t1, ~ST0_CU1
and t0, t0, t1 and t0, t0, t1
LONG_S t0, ST_OFF(t3) LONG_S t0, ST_OFF(t3)
/* clear thread_struct CU1 bit */
and t2, t1
fpu_save_double a0 t0 t1 # c0_status passed in t0 fpu_save_double a0 t0 t1 # c0_status passed in t0
# clobbers t1 # clobbers t1
1: 1:
LONG_S t2, THREAD_STATUS(a0)
/* /*
* The order of restoring the registers takes care of the race * The order of restoring the registers takes care of the race

29
arch/mips/kernel/signal.c
View File

@ -82,6 +82,7 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
{ {
int err = 0; int err = 0;
int i; int i;
unsigned int used_math;
err |= __put_user(regs->cp0_epc, &sc->sc_pc); err |= __put_user(regs->cp0_epc, &sc->sc_pc);
@ -104,22 +105,18 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp); err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp);
} }
err |= __put_user(!!used_math(), &sc->sc_used_math); used_math = !!used_math();
err |= __put_user(used_math, &sc->sc_used_math);
if (used_math()) { if (used_math) {
/* /*
* Save FPU state to signal context. Signal handler * Save FPU state to signal context. Signal handler
* will "inherit" current FPU state. * will "inherit" current FPU state.
*/ */
preempt_disable(); own_fpu(1);
enable_fp_in_kernel();
if (!is_fpu_owner()) {
own_fpu();
restore_fp(current);
}
err |= save_fp_context(sc); err |= save_fp_context(sc);
disable_fp_in_kernel();
preempt_enable();
} }
return err; return err;
} }
@ -188,20 +185,18 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
err |= __get_user(used_math, &sc->sc_used_math); err |= __get_user(used_math, &sc->sc_used_math);
conditional_used_math(used_math); conditional_used_math(used_math);
preempt_disable(); if (used_math) {
if (used_math()) {
/* restore fpu context if we have used it before */ /* restore fpu context if we have used it before */
own_fpu(); own_fpu(0);
enable_fp_in_kernel();
if (!err) if (!err)
err = check_and_restore_fp_context(sc); err = check_and_restore_fp_context(sc);
disable_fp_in_kernel();
} else { } else {
/* signal handler may have used FPU. Give it up. */ /* signal handler may have used FPU. Give it up. */
lose_fpu(); lose_fpu(0);
} }
preempt_enable();
return err; return err;
} }

29
arch/mips/kernel/signal32.c
View File

@ -181,6 +181,7 @@ static int setup_sigcontext32(struct pt_regs *regs,
{ {
int err = 0; int err = 0;
int i; int i;
u32 used_math;
err |= __put_user(regs->cp0_epc, &sc->sc_pc); err |= __put_user(regs->cp0_epc, &sc->sc_pc);
@ -200,22 +201,18 @@ static int setup_sigcontext32(struct pt_regs *regs,
err |= __put_user(mflo3(), &sc->sc_lo3); err |= __put_user(mflo3(), &sc->sc_lo3);
} }
err |= __put_user(!!used_math(), &sc->sc_used_math); used_math = !!used_math();
err |= __put_user(used_math, &sc->sc_used_math);
if (used_math()) { if (used_math) {
/* /*
* Save FPU state to signal context. Signal handler * Save FPU state to signal context. Signal handler
* will "inherit" current FPU state. * will "inherit" current FPU state.
*/ */
preempt_disable(); own_fpu(1);
enable_fp_in_kernel();
if (!is_fpu_owner()) {
own_fpu();
restore_fp(current);
}
err |= save_fp_context32(sc); err |= save_fp_context32(sc);
disable_fp_in_kernel();
preempt_enable();
} }
return err; return err;
} }
@ -262,20 +259,18 @@ static int restore_sigcontext32(struct pt_regs *regs,
err |= __get_user(used_math, &sc->sc_used_math); err |= __get_user(used_math, &sc->sc_used_math);
conditional_used_math(used_math); conditional_used_math(used_math);
preempt_disable(); if (used_math) {
if (used_math()) {
/* restore fpu context if we have used it before */ /* restore fpu context if we have used it before */
own_fpu(); own_fpu(0);
enable_fp_in_kernel();
if (!err) if (!err)
err = check_and_restore_fp_context32(sc); err = check_and_restore_fp_context32(sc);
disable_fp_in_kernel();
} else { } else {
/* signal handler may have used FPU. Give it up. */ /* signal handler may have used FPU. Give it up. */
lose_fpu(); lose_fpu(0);
} }
preempt_enable();
return err; return err;
} }

84
arch/mips/kernel/traps.c
View File

@ -610,16 +610,6 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
if (fcr31 & FPU_CSR_UNI_X) { if (fcr31 & FPU_CSR_UNI_X) {
int sig; int sig;
preempt_disable();
#ifdef CONFIG_PREEMPT
if (!is_fpu_owner()) {
/* We might lose fpu before disabling preempt... */
own_fpu();
BUG_ON(!used_math());
restore_fp(current);
}
#endif
/* /*
* Unimplemented operation exception. If we've got the full * Unimplemented operation exception. If we've got the full
* software emulator on-board, let's use it... * software emulator on-board, let's use it...
@ -630,18 +620,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
* register operands before invoking the emulator, which seems * register operands before invoking the emulator, which seems
* a bit extreme for what should be an infrequent event. * a bit extreme for what should be an infrequent event.
*/ */
save_fp(current);
/* Ensure 'resume' not overwrite saved fp context again. */ /* Ensure 'resume' not overwrite saved fp context again. */
lose_fpu(); lose_fpu(1);
preempt_enable();
/* Run the emulator */ /* Run the emulator */
sig = fpu_emulator_cop1Handler (regs, &current->thread.fpu, 1); sig = fpu_emulator_cop1Handler (regs, &current->thread.fpu, 1);
preempt_disable();
own_fpu(); /* Using the FPU again. */
/* /*
* We can't allow the emulated instruction to leave any of * We can't allow the emulated instruction to leave any of
* the cause bit set in $fcr31. * the cause bit set in $fcr31.
@ -649,9 +633,7 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
/* Restore the hardware register state */ /* Restore the hardware register state */
restore_fp(current); own_fpu(1); /* Using the FPU again. */
preempt_enable();
/* If something went wrong, signal */ /* If something went wrong, signal */
if (sig) if (sig)
@ -775,12 +757,11 @@ asmlinkage void do_cpu(struct pt_regs *regs)
{ {
unsigned int cpid; unsigned int cpid;
die_if_kernel("do_cpu invoked from kernel context!", regs);
cpid = (regs->cp0_cause >> CAUSEB_CE) & 3; cpid = (regs->cp0_cause >> CAUSEB_CE) & 3;
switch (cpid) { switch (cpid) {
case 0: case 0:
die_if_kernel("do_cpu invoked from kernel context!", regs);
if (!cpu_has_llsc) if (!cpu_has_llsc)
if (!simulate_llsc(regs)) if (!simulate_llsc(regs))
return; return;
@ -791,21 +772,30 @@ asmlinkage void do_cpu(struct pt_regs *regs)
break; break;
case 1: case 1:
preempt_disable(); if (!test_thread_flag(TIF_ALLOW_FP_IN_KERNEL))
die_if_kernel("do_cpu invoked from kernel context!",
own_fpu(); regs);
if (used_math()) { /* Using the FPU again. */ if (used_math()) /* Using the FPU again. */
restore_fp(current); own_fpu(1);
} else { /* First time FPU user. */ else { /* First time FPU user. */
init_fpu(); init_fpu();
set_used_math(); set_used_math();
} }
if (cpu_has_fpu) { if (raw_cpu_has_fpu) {
preempt_enable(); if (test_thread_flag(TIF_ALLOW_FP_IN_KERNEL)) {
local_irq_disable();
if (cpu_has_fpu)
regs->cp0_status |= ST0_CU1;
/*
* We must return without enabling
* interrupts to ensure keep FPU
* ownership until resume.
*/
return;
}
} else { } else {
int sig; int sig;
preempt_enable();
sig = fpu_emulator_cop1Handler(regs, sig = fpu_emulator_cop1Handler(regs,
&current->thread.fpu, 0); &current->thread.fpu, 0);
if (sig) if (sig)
@ -1259,26 +1249,26 @@ static inline void mips_srs_init(void)
/* /*
* This is used by native signal handling * This is used by native signal handling
*/ */
asmlinkage int (*save_fp_context)(struct sigcontext *sc); asmlinkage int (*save_fp_context)(struct sigcontext __user *sc);
asmlinkage int (*restore_fp_context)(struct sigcontext *sc); asmlinkage int (*restore_fp_context)(struct sigcontext __user *sc);
extern asmlinkage int _save_fp_context(struct sigcontext *sc); extern asmlinkage int _save_fp_context(struct sigcontext __user *sc);
extern asmlinkage int _restore_fp_context(struct sigcontext *sc); extern asmlinkage int _restore_fp_context(struct sigcontext __user *sc);
extern asmlinkage int fpu_emulator_save_context(struct sigcontext *sc); extern asmlinkage int fpu_emulator_save_context(struct sigcontext __user *sc);
extern asmlinkage int fpu_emulator_restore_context(struct sigcontext *sc); extern asmlinkage int fpu_emulator_restore_context(struct sigcontext __user *sc);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static int smp_save_fp_context(struct sigcontext *sc) static int smp_save_fp_context(struct sigcontext __user *sc)
{ {
return cpu_has_fpu return raw_cpu_has_fpu
? _save_fp_context(sc) ? _save_fp_context(sc)
: fpu_emulator_save_context(sc); : fpu_emulator_save_context(sc);
} }
static int smp_restore_fp_context(struct sigcontext *sc) static int smp_restore_fp_context(struct sigcontext __user *sc)
{ {
return cpu_has_fpu return raw_cpu_has_fpu
? _restore_fp_context(sc) ? _restore_fp_context(sc)
: fpu_emulator_restore_context(sc); : fpu_emulator_restore_context(sc);
} }
@ -1306,14 +1296,14 @@ static inline void signal_init(void)
/* /*
* This is used by 32-bit signal stuff on the 64-bit kernel * This is used by 32-bit signal stuff on the 64-bit kernel
*/ */
asmlinkage int (*save_fp_context32)(struct sigcontext32 *sc); asmlinkage int (*save_fp_context32)(struct sigcontext32 __user *sc);
asmlinkage int (*restore_fp_context32)(struct sigcontext32 *sc); asmlinkage int (*restore_fp_context32)(struct sigcontext32 __user *sc);
extern asmlinkage int _save_fp_context32(struct sigcontext32 *sc); extern asmlinkage int _save_fp_context32(struct sigcontext32 __user *sc);
extern asmlinkage int _restore_fp_context32(struct sigcontext32 *sc); extern asmlinkage int _restore_fp_context32(struct sigcontext32 __user *sc);
extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 *sc); extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 __user *sc);
extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 *sc); extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 __user *sc);
static inline void signal32_init(void) static inline void signal32_init(void)
{ {

8
arch/mips/math-emu/kernel_linkage.c
View File

@ -51,7 +51,7 @@ void fpu_emulator_init_fpu(void)
* with appropriate macros from uaccess.h * with appropriate macros from uaccess.h
*/ */
int fpu_emulator_save_context(struct sigcontext *sc) int fpu_emulator_save_context(struct sigcontext __user *sc)
{ {
int i; int i;
int err = 0; int err = 0;
@ -65,7 +65,7 @@ int fpu_emulator_save_context(struct sigcontext *sc)
return err; return err;
} }
int fpu_emulator_restore_context(struct sigcontext *sc) int fpu_emulator_restore_context(struct sigcontext __user *sc)
{ {
int i; int i;
int err = 0; int err = 0;
@ -84,7 +84,7 @@ int fpu_emulator_restore_context(struct sigcontext *sc)
* This is the o32 version * This is the o32 version
*/ */
int fpu_emulator_save_context32(struct sigcontext32 *sc) int fpu_emulator_save_context32(struct sigcontext32 __user *sc)
{ {
int i; int i;
int err = 0; int err = 0;
@ -98,7 +98,7 @@ int fpu_emulator_save_context32(struct sigcontext32 *sc)
return err; return err;
} }
int fpu_emulator_restore_context32(struct sigcontext32 *sc) int fpu_emulator_restore_context32(struct sigcontext32 __user *sc)
{ {
int i; int i;
int err = 0; int err = 0;

3
include/asm-mips/cpu-features.h
View File

@ -40,6 +40,9 @@
#endif #endif
#ifndef cpu_has_fpu #ifndef cpu_has_fpu
#define cpu_has_fpu (current_cpu_data.options & MIPS_CPU_FPU) #define cpu_has_fpu (current_cpu_data.options & MIPS_CPU_FPU)
#define raw_cpu_has_fpu (raw_current_cpu_data.options & MIPS_CPU_FPU)
#else
#define raw_cpu_has_fpu cpu_has_fpu
#endif #endif
#ifndef cpu_has_32fpr #ifndef cpu_has_32fpr
#define cpu_has_32fpr (cpu_data[0].options & MIPS_CPU_32FPR) #define cpu_has_32fpr (cpu_data[0].options & MIPS_CPU_32FPR)

1
include/asm-mips/cpu-info.h
View File

@ -87,6 +87,7 @@ struct cpuinfo_mips {
extern struct cpuinfo_mips cpu_data[]; extern struct cpuinfo_mips cpu_data[];
#define current_cpu_data cpu_data[smp_processor_id()] #define current_cpu_data cpu_data[smp_processor_id()]
#define raw_current_cpu_data cpu_data[raw_smp_processor_id()]
extern void cpu_probe(void); extern void cpu_probe(void);
extern void cpu_report(void); extern void cpu_report(void);

56
include/asm-mips/fpu.h
View File

@ -27,11 +27,11 @@
struct sigcontext; struct sigcontext;
struct sigcontext32; struct sigcontext32;
extern asmlinkage int (*save_fp_context)(struct sigcontext *sc); extern asmlinkage int (*save_fp_context)(struct sigcontext __user *sc);
extern asmlinkage int (*restore_fp_context)(struct sigcontext *sc); extern asmlinkage int (*restore_fp_context)(struct sigcontext __user *sc);
extern asmlinkage int (*save_fp_context32)(struct sigcontext32 *sc); extern asmlinkage int (*save_fp_context32)(struct sigcontext32 __user *sc);
extern asmlinkage int (*restore_fp_context32)(struct sigcontext32 *sc); extern asmlinkage int (*restore_fp_context32)(struct sigcontext32 __user *sc);
extern void fpu_emulator_init_fpu(void); extern void fpu_emulator_init_fpu(void);
extern void _init_fpu(void); extern void _init_fpu(void);
@ -68,6 +68,8 @@ do { \
/* We don't care about the c0 hazard here */ \ /* We don't care about the c0 hazard here */ \
} while (0) } while (0)
#define __fpu_enabled() (read_c0_status() & ST0_CU1)
#define enable_fpu() \ #define enable_fpu() \
do { \ do { \
if (cpu_has_fpu) \ if (cpu_has_fpu) \
@ -93,31 +95,47 @@ static inline int is_fpu_owner(void)
return cpu_has_fpu && __is_fpu_owner(); return cpu_has_fpu && __is_fpu_owner();
} }
static inline void own_fpu(void) static inline void __own_fpu(void)
{ {
if (cpu_has_fpu) { __enable_fpu();
__enable_fpu(); KSTK_STATUS(current) |= ST0_CU1;
KSTK_STATUS(current) |= ST0_CU1; set_thread_flag(TIF_USEDFPU);
set_thread_flag(TIF_USEDFPU);
}
} }
static inline void lose_fpu(void) static inline void own_fpu(int restore)
{ {
if (cpu_has_fpu) { preempt_disable();
if (cpu_has_fpu && !__is_fpu_owner()) {
__own_fpu();
if (restore)
_restore_fp(current);
}
preempt_enable();
}
static inline void lose_fpu(int save)
{
preempt_disable();
if (is_fpu_owner()) {
if (save)
_save_fp(current);
KSTK_STATUS(current) &= ~ST0_CU1; KSTK_STATUS(current) &= ~ST0_CU1;
clear_thread_flag(TIF_USEDFPU); clear_thread_flag(TIF_USEDFPU);
__disable_fpu(); __disable_fpu();
} }
preempt_enable();
} }
static inline void init_fpu(void) static inline void init_fpu(void)
{ {
preempt_disable();
if (cpu_has_fpu) { if (cpu_has_fpu) {
__own_fpu();
_init_fpu(); _init_fpu();
} else { } else {
fpu_emulator_init_fpu(); fpu_emulator_init_fpu();
} }
preempt_enable();
} }
static inline void save_fp(struct task_struct *tsk) static inline void save_fp(struct task_struct *tsk)
@ -144,4 +162,18 @@ static inline fpureg_t *get_fpu_regs(struct task_struct *tsk)
return tsk->thread.fpu.fpr; return tsk->thread.fpu.fpr;
} }
static inline void enable_fp_in_kernel(void)
{
set_thread_flag(TIF_ALLOW_FP_IN_KERNEL);
/* make sure CU1 and FPU ownership are consistent */
if (!__is_fpu_owner() && __fpu_enabled())
__disable_fpu();
}
static inline void disable_fp_in_kernel(void)
{
BUG_ON(!__is_fpu_owner() && __fpu_enabled());
clear_thread_flag(TIF_ALLOW_FP_IN_KERNEL);
}
#endif /* _ASM_FPU_H */ #endif /* _ASM_FPU_H */

1
include/asm-mips/thread_info.h
View File

@ -119,6 +119,7 @@ register struct thread_info *__current_thread_info __asm__("$28");
#define TIF_POLLING_NRFLAG 17 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_POLLING_NRFLAG 17 /* true if poll_idle() is polling TIF_NEED_RESCHED */
#define TIF_MEMDIE 18 #define TIF_MEMDIE 18
#define TIF_FREEZE 19 #define TIF_FREEZE 19
#define TIF_ALLOW_FP_IN_KERNEL 20
#define TIF_SYSCALL_TRACE 31 /* syscall trace active */ #define TIF_SYSCALL_TRACE 31 /* syscall trace active */
#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)