x86-64: Slightly shorten line system call entry and exit paths
GET_THREAD_INFO() involves a memory read immediately followed by an "sub" on the value read, in turn (in several cases) immediately followed by a use of the calculated value as the base address of a memory access. This combination of instructions has a non-negligible potential for stalls. In the system call entry point code, however, the (fixed) offset of the stack pointer from the end of the stack is generally known, and hence we can instead avoid the memory load and subtract, and instead do the memory reference using %rsp as the base register. To do so in a legible fashion, introduce a THREAD_INFO() macro which, provided a register (generally %rsp) and the known offset from the end of the stack, produces a suitable memory access operand. The patch attempts to only touch the fast paths (no auditing and alike), but manages to do so only in the 64-bit entry point case; the compatibility mode entry points have so many interdependencies between their various branch targets that it was necessary to also adjust the slow paths to eliminate the risk of having missed some register dependency during code analysis. Signed-off-by: Jan Beulich <jbeulich@suse.com> Reviewed-by: Andi Kleen <ak@linux.intel.com> Link: http://lkml.kernel.org/r/4ED4CD690200007800064075@nat28.tlf.novell.com Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Jan Beulich
Ingo Molnar
parent
39e9543344
commit
46db09d3fd
|
|
@ -134,7 +134,7 @@ ENTRY(ia32_sysenter_target)
|
|||
CFI_REL_OFFSET rsp,0
|
||||
pushfq_cfi
|
||||
/*CFI_REL_OFFSET rflags,0*/
|
||||
movl 8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d
|
||||
movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
|
||||
CFI_REGISTER rip,r10
|
||||
pushq_cfi $__USER32_CS
|
||||
/*CFI_REL_OFFSET cs,0*/
|
||||
|
|
@ -150,9 +150,8 @@ ENTRY(ia32_sysenter_target)
|
|||
.section __ex_table,"a"
|
||||
.quad 1b,ia32_badarg
|
||||
.previous
|
||||
GET_THREAD_INFO(%r10)
|
||||
orl $TS_COMPAT,TI_status(%r10)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
|
||||
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
CFI_REMEMBER_STATE
|
||||
jnz sysenter_tracesys
|
||||
cmpq $(IA32_NR_syscalls-1),%rax
|
||||
|
|
@ -162,13 +161,12 @@ sysenter_do_call:
|
|||
sysenter_dispatch:
|
||||
call *ia32_sys_call_table(,%rax,8)
|
||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
||||
GET_THREAD_INFO(%r10)
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF
|
||||
testl $_TIF_ALLWORK_MASK,TI_flags(%r10)
|
||||
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jnz sysexit_audit
|
||||
sysexit_from_sys_call:
|
||||
andl $~TS_COMPAT,TI_status(%r10)
|
||||
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
/* clear IF, that popfq doesn't enable interrupts early */
|
||||
andl $~0x200,EFLAGS-R11(%rsp)
|
||||
movl RIP-R11(%rsp),%edx /* User %eip */
|
||||
|
|
@ -205,7 +203,7 @@ sysexit_from_sys_call:
|
|||
.endm
|
||||
|
||||
.macro auditsys_exit exit
|
||||
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)
|
||||
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jnz ia32_ret_from_sys_call
|
||||
TRACE_IRQS_ON
|
||||
sti
|
||||
|
|
@ -215,12 +213,11 @@ sysexit_from_sys_call:
|
|||
movzbl %al,%edi /* zero-extend that into %edi */
|
||||
inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
|
||||
call audit_syscall_exit
|
||||
GET_THREAD_INFO(%r10)
|
||||
movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall return value */
|
||||
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
|
||||
cli
|
||||
TRACE_IRQS_OFF
|
||||
testl %edi,TI_flags(%r10)
|
||||
testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jz \exit
|
||||
CLEAR_RREGS -ARGOFFSET
|
||||
jmp int_with_check
|
||||
|
|
@ -238,7 +235,7 @@ sysexit_audit:
|
|||
|
||||
sysenter_tracesys:
|
||||
#ifdef CONFIG_AUDITSYSCALL
|
||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)
|
||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jz sysenter_auditsys
|
||||
#endif
|
||||
SAVE_REST
|
||||
|
|
@ -309,9 +306,8 @@ ENTRY(ia32_cstar_target)
|
|||
.section __ex_table,"a"
|
||||
.quad 1b,ia32_badarg
|
||||
.previous
|
||||
GET_THREAD_INFO(%r10)
|
||||
orl $TS_COMPAT,TI_status(%r10)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
|
||||
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
CFI_REMEMBER_STATE
|
||||
jnz cstar_tracesys
|
||||
cmpq $IA32_NR_syscalls-1,%rax
|
||||
|
|
@ -321,13 +317,12 @@ cstar_do_call:
|
|||
cstar_dispatch:
|
||||
call *ia32_sys_call_table(,%rax,8)
|
||||
movq %rax,RAX-ARGOFFSET(%rsp)
|
||||
GET_THREAD_INFO(%r10)
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF
|
||||
testl $_TIF_ALLWORK_MASK,TI_flags(%r10)
|
||||
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jnz sysretl_audit
|
||||
sysretl_from_sys_call:
|
||||
andl $~TS_COMPAT,TI_status(%r10)
|
||||
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
RESTORE_ARGS 0,-ARG_SKIP,0,0,0
|
||||
movl RIP-ARGOFFSET(%rsp),%ecx
|
||||
CFI_REGISTER rip,rcx
|
||||
|
|
@ -355,7 +350,7 @@ sysretl_audit:
|
|||
|
||||
cstar_tracesys:
|
||||
#ifdef CONFIG_AUDITSYSCALL
|
||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)
|
||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jz cstar_auditsys
|
||||
#endif
|
||||
xchgl %r9d,%ebp
|
||||
|
|
@ -420,9 +415,8 @@ ENTRY(ia32_syscall)
|
|||
/* note the registers are not zero extended to the sf.
|
||||
this could be a problem. */
|
||||
SAVE_ARGS 0,1,0
|
||||
GET_THREAD_INFO(%r10)
|
||||
orl $TS_COMPAT,TI_status(%r10)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
|
||||
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jnz ia32_tracesys
|
||||
cmpq $(IA32_NR_syscalls-1),%rax
|
||||
ja ia32_badsys
|
||||
|
|
|
|||
|
|
@ -232,6 +232,12 @@ static inline struct thread_info *current_thread_info(void)
|
|||
movq PER_CPU_VAR(kernel_stack),reg ; \
|
||||
subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
|
||||
|
||||
/*
|
||||
* Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
|
||||
* a certain register (to be used in assembler memory operands).
|
||||
*/
|
||||
#define THREAD_INFO(reg, off) KERNEL_STACK_OFFSET+(off)-THREAD_SIZE(reg)
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* !X86_32 */
|
||||
|
|
|
|||
|
|
@ -478,8 +478,7 @@ ENTRY(system_call_after_swapgs)
|
|||
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
|
||||
movq %rcx,RIP-ARGOFFSET(%rsp)
|
||||
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
||||
GET_THREAD_INFO(%rcx)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jnz tracesys
|
||||
system_call_fastpath:
|
||||
cmpq $__NR_syscall_max,%rax
|
||||
|
|
@ -496,10 +495,9 @@ ret_from_sys_call:
|
|||
/* edi: flagmask */
|
||||
sysret_check:
|
||||
LOCKDEP_SYS_EXIT
|
||||
GET_THREAD_INFO(%rcx)
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF
|
||||
movl TI_flags(%rcx),%edx
|
||||
movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx
|
||||
andl %edi,%edx
|
||||
jnz sysret_careful
|
||||
CFI_REMEMBER_STATE
|
||||
|
|
@ -583,7 +581,7 @@ sysret_audit:
|
|||
/* Do syscall tracing */
|
||||
tracesys:
|
||||
#ifdef CONFIG_AUDITSYSCALL
|
||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
|
||||
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
|
||||
jz auditsys
|
||||
#endif
|
||||
SAVE_REST
|
||||
|
|
|
|||
Loading…
Reference in New Issue