s390: run user space and KVM guests with modified branch prediction
Define TIF_ISOLATE_BP and TIF_ISOLATE_BP_GUEST and add the necessary plumbing in entry.S to be able to run user space and KVM guests with limited branch prediction. To switch a user space process to limited branch prediction the s390_isolate_bp() function has to be call, and to run a vCPU of a KVM guest associated with the current task with limited branch prediction call s390_isolate_bp_guest(). Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
This commit is contained in:
parent
d768bd892f
commit
6b73044b2b
|
@ -378,6 +378,9 @@ extern void memcpy_absolute(void *, void *, size_t);
|
||||||
memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \
|
memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
|
extern int s390_isolate_bp(void);
|
||||||
|
extern int s390_isolate_bp_guest(void);
|
||||||
|
|
||||||
#endif /* __ASSEMBLY__ */
|
#endif /* __ASSEMBLY__ */
|
||||||
|
|
||||||
#endif /* __ASM_S390_PROCESSOR_H */
|
#endif /* __ASM_S390_PROCESSOR_H */
|
||||||
|
|
|
@ -58,6 +58,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
|
||||||
#define TIF_GUARDED_STORAGE 4 /* load guarded storage control block */
|
#define TIF_GUARDED_STORAGE 4 /* load guarded storage control block */
|
||||||
#define TIF_PATCH_PENDING 5 /* pending live patching update */
|
#define TIF_PATCH_PENDING 5 /* pending live patching update */
|
||||||
#define TIF_PGSTE 6 /* New mm's will use 4K page tables */
|
#define TIF_PGSTE 6 /* New mm's will use 4K page tables */
|
||||||
|
#define TIF_ISOLATE_BP 8 /* Run process with isolated BP */
|
||||||
|
#define TIF_ISOLATE_BP_GUEST 9 /* Run KVM guests with isolated BP */
|
||||||
|
|
||||||
#define TIF_31BIT 16 /* 32bit process */
|
#define TIF_31BIT 16 /* 32bit process */
|
||||||
#define TIF_MEMDIE 17 /* is terminating due to OOM killer */
|
#define TIF_MEMDIE 17 /* is terminating due to OOM killer */
|
||||||
|
@ -78,6 +80,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
|
||||||
#define _TIF_UPROBE _BITUL(TIF_UPROBE)
|
#define _TIF_UPROBE _BITUL(TIF_UPROBE)
|
||||||
#define _TIF_GUARDED_STORAGE _BITUL(TIF_GUARDED_STORAGE)
|
#define _TIF_GUARDED_STORAGE _BITUL(TIF_GUARDED_STORAGE)
|
||||||
#define _TIF_PATCH_PENDING _BITUL(TIF_PATCH_PENDING)
|
#define _TIF_PATCH_PENDING _BITUL(TIF_PATCH_PENDING)
|
||||||
|
#define _TIF_ISOLATE_BP _BITUL(TIF_ISOLATE_BP)
|
||||||
|
#define _TIF_ISOLATE_BP_GUEST _BITUL(TIF_ISOLATE_BP_GUEST)
|
||||||
|
|
||||||
#define _TIF_31BIT _BITUL(TIF_31BIT)
|
#define _TIF_31BIT _BITUL(TIF_31BIT)
|
||||||
#define _TIF_SINGLE_STEP _BITUL(TIF_SINGLE_STEP)
|
#define _TIF_SINGLE_STEP _BITUL(TIF_SINGLE_STEP)
|
||||||
|
|
|
@ -107,6 +107,7 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
|
||||||
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
||||||
j 3f
|
j 3f
|
||||||
1: UPDATE_VTIME %r14,%r15,\timer
|
1: UPDATE_VTIME %r14,%r15,\timer
|
||||||
|
BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
|
||||||
2: lg %r15,__LC_ASYNC_STACK # load async stack
|
2: lg %r15,__LC_ASYNC_STACK # load async stack
|
||||||
3: la %r11,STACK_FRAME_OVERHEAD(%r15)
|
3: la %r11,STACK_FRAME_OVERHEAD(%r15)
|
||||||
.endm
|
.endm
|
||||||
|
@ -187,6 +188,40 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
|
||||||
.popsection
|
.popsection
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
.macro BPENTER tif_ptr,tif_mask
|
||||||
|
.pushsection .altinstr_replacement, "ax"
|
||||||
|
662: .word 0xc004, 0x0000, 0x0000 # 6 byte nop
|
||||||
|
.word 0xc004, 0x0000, 0x0000 # 6 byte nop
|
||||||
|
.popsection
|
||||||
|
664: TSTMSK \tif_ptr,\tif_mask
|
||||||
|
jz . + 8
|
||||||
|
.long 0xb2e8d000
|
||||||
|
.pushsection .altinstructions, "a"
|
||||||
|
.long 664b - .
|
||||||
|
.long 662b - .
|
||||||
|
.word 82
|
||||||
|
.byte 12
|
||||||
|
.byte 12
|
||||||
|
.popsection
|
||||||
|
.endm
|
||||||
|
|
||||||
|
.macro BPEXIT tif_ptr,tif_mask
|
||||||
|
TSTMSK \tif_ptr,\tif_mask
|
||||||
|
.pushsection .altinstr_replacement, "ax"
|
||||||
|
662: jnz . + 8
|
||||||
|
.long 0xb2e8d000
|
||||||
|
.popsection
|
||||||
|
664: jz . + 8
|
||||||
|
.long 0xb2e8c000
|
||||||
|
.pushsection .altinstructions, "a"
|
||||||
|
.long 664b - .
|
||||||
|
.long 662b - .
|
||||||
|
.word 82
|
||||||
|
.byte 8
|
||||||
|
.byte 8
|
||||||
|
.popsection
|
||||||
|
.endm
|
||||||
|
|
||||||
.section .kprobes.text, "ax"
|
.section .kprobes.text, "ax"
|
||||||
.Ldummy:
|
.Ldummy:
|
||||||
/*
|
/*
|
||||||
|
@ -240,9 +275,11 @@ ENTRY(__switch_to)
|
||||||
*/
|
*/
|
||||||
ENTRY(sie64a)
|
ENTRY(sie64a)
|
||||||
stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
|
stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
|
||||||
|
lg %r12,__LC_CURRENT
|
||||||
stg %r2,__SF_EMPTY(%r15) # save control block pointer
|
stg %r2,__SF_EMPTY(%r15) # save control block pointer
|
||||||
stg %r3,__SF_EMPTY+8(%r15) # save guest register save area
|
stg %r3,__SF_EMPTY+8(%r15) # save guest register save area
|
||||||
xc __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
|
xc __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
|
||||||
|
mvc __SF_EMPTY+24(8,%r15),__TI_flags(%r12) # copy thread flags
|
||||||
TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ?
|
TSTMSK __LC_CPU_FLAGS,_CIF_FPU # load guest fp/vx registers ?
|
||||||
jno .Lsie_load_guest_gprs
|
jno .Lsie_load_guest_gprs
|
||||||
brasl %r14,load_fpu_regs # load guest fp/vx regs
|
brasl %r14,load_fpu_regs # load guest fp/vx regs
|
||||||
|
@ -259,11 +296,12 @@ ENTRY(sie64a)
|
||||||
jnz .Lsie_skip
|
jnz .Lsie_skip
|
||||||
TSTMSK __LC_CPU_FLAGS,_CIF_FPU
|
TSTMSK __LC_CPU_FLAGS,_CIF_FPU
|
||||||
jo .Lsie_skip # exit if fp/vx regs changed
|
jo .Lsie_skip # exit if fp/vx regs changed
|
||||||
BPON
|
BPEXIT __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
|
||||||
.Lsie_entry:
|
.Lsie_entry:
|
||||||
sie 0(%r14)
|
sie 0(%r14)
|
||||||
.Lsie_exit:
|
.Lsie_exit:
|
||||||
BPOFF
|
BPOFF
|
||||||
|
BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
|
||||||
.Lsie_skip:
|
.Lsie_skip:
|
||||||
ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
|
ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
|
||||||
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
||||||
|
@ -324,6 +362,7 @@ ENTRY(system_call)
|
||||||
la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
|
la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
|
||||||
.Lsysc_vtime:
|
.Lsysc_vtime:
|
||||||
UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
|
UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
|
||||||
|
BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
|
||||||
stmg %r0,%r7,__PT_R0(%r11)
|
stmg %r0,%r7,__PT_R0(%r11)
|
||||||
# clear user controlled register to prevent speculative use
|
# clear user controlled register to prevent speculative use
|
||||||
xgr %r0,%r0
|
xgr %r0,%r0
|
||||||
|
@ -362,7 +401,7 @@ ENTRY(system_call)
|
||||||
jnz .Lsysc_work # check for work
|
jnz .Lsysc_work # check for work
|
||||||
TSTMSK __LC_CPU_FLAGS,_CIF_WORK
|
TSTMSK __LC_CPU_FLAGS,_CIF_WORK
|
||||||
jnz .Lsysc_work
|
jnz .Lsysc_work
|
||||||
BPON
|
BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
|
||||||
.Lsysc_restore:
|
.Lsysc_restore:
|
||||||
lg %r14,__LC_VDSO_PER_CPU
|
lg %r14,__LC_VDSO_PER_CPU
|
||||||
lmg %r0,%r10,__PT_R0(%r11)
|
lmg %r0,%r10,__PT_R0(%r11)
|
||||||
|
@ -597,6 +636,7 @@ ENTRY(pgm_check_handler)
|
||||||
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
|
||||||
j 4f
|
j 4f
|
||||||
2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
|
2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
|
||||||
|
BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
|
||||||
lg %r15,__LC_KERNEL_STACK
|
lg %r15,__LC_KERNEL_STACK
|
||||||
lgr %r14,%r12
|
lgr %r14,%r12
|
||||||
aghi %r14,__TASK_thread # pointer to thread_struct
|
aghi %r14,__TASK_thread # pointer to thread_struct
|
||||||
|
@ -729,7 +769,7 @@ ENTRY(io_int_handler)
|
||||||
mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
|
mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
|
||||||
tm __PT_PSW+1(%r11),0x01 # returning to user ?
|
tm __PT_PSW+1(%r11),0x01 # returning to user ?
|
||||||
jno .Lio_exit_kernel
|
jno .Lio_exit_kernel
|
||||||
BPON
|
BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
|
||||||
.Lio_exit_timer:
|
.Lio_exit_timer:
|
||||||
stpt __LC_EXIT_TIMER
|
stpt __LC_EXIT_TIMER
|
||||||
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
||||||
|
@ -1165,7 +1205,7 @@ ENTRY(mcck_int_handler)
|
||||||
mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
|
mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
|
||||||
tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
|
tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
|
||||||
jno 0f
|
jno 0f
|
||||||
BPON
|
BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
|
||||||
stpt __LC_EXIT_TIMER
|
stpt __LC_EXIT_TIMER
|
||||||
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
|
||||||
0: lmg %r11,%r15,__PT_R11(%r11)
|
0: lmg %r11,%r15,__PT_R11(%r11)
|
||||||
|
@ -1292,7 +1332,8 @@ cleanup_critical:
|
||||||
clg %r9,BASED(.Lsie_crit_mcck_length)
|
clg %r9,BASED(.Lsie_crit_mcck_length)
|
||||||
jh 1f
|
jh 1f
|
||||||
oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
|
oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
|
||||||
1: lg %r9,__SF_EMPTY(%r15) # get control block pointer
|
1: BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
|
||||||
|
lg %r9,__SF_EMPTY(%r15) # get control block pointer
|
||||||
ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
|
ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
|
||||||
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
|
||||||
larl %r9,sie_exit # skip forward to sie_exit
|
larl %r9,sie_exit # skip forward to sie_exit
|
||||||
|
|
|
@ -197,3 +197,21 @@ const struct seq_operations cpuinfo_op = {
|
||||||
.stop = c_stop,
|
.stop = c_stop,
|
||||||
.show = show_cpuinfo,
|
.show = show_cpuinfo,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
int s390_isolate_bp(void)
|
||||||
|
{
|
||||||
|
if (!test_facility(82))
|
||||||
|
return -EOPNOTSUPP;
|
||||||
|
set_thread_flag(TIF_ISOLATE_BP);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(s390_isolate_bp);
|
||||||
|
|
||||||
|
int s390_isolate_bp_guest(void)
|
||||||
|
{
|
||||||
|
if (!test_facility(82))
|
||||||
|
return -EOPNOTSUPP;
|
||||||
|
set_thread_flag(TIF_ISOLATE_BP_GUEST);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL(s390_isolate_bp_guest);
|
||||||
|
|
Loading…
Reference in New Issue