2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* Low-level exception handling code
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
* Authors: Catalin Marinas <catalin.marinas@arm.com>
|
|
|
|
|
* Will Deacon <will.deacon@arm.com>
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
|
|
2015-06-01 17:47:41 +08:00
|
|
|
#include <asm/alternative.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/assembler.h>
|
|
|
|
|
#include <asm/asm-offsets.h>
|
2015-03-24 03:07:02 +08:00
|
|
|
#include <asm/cpufeature.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/errno.h>
|
2013-04-09 00:17:03 +08:00
|
|
|
#include <asm/esr.h>
|
2015-12-04 19:02:27 +08:00
|
|
|
#include <asm/irq.h>
|
2016-06-21 01:28:01 +08:00
|
|
|
#include <asm/memory.h>
|
2016-09-02 21:54:03 +08:00
|
|
|
#include <asm/ptrace.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/thread_info.h>
|
2016-12-26 17:10:19 +08:00
|
|
|
#include <asm/asm-uaccess.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/unistd.h>
|
|
|
|
|
|
2014-05-31 03:34:15 +08:00
|
|
|
/*
|
|
|
|
|
* Context tracking subsystem. Used to instrument transitions
|
|
|
|
|
* between user and kernel mode.
|
|
|
|
|
*/
|
|
|
|
|
.macro ct_user_exit, syscall = 0
|
|
|
|
|
#ifdef CONFIG_CONTEXT_TRACKING
|
|
|
|
|
bl context_tracking_user_exit
|
|
|
|
|
.if \syscall == 1
|
|
|
|
|
/*
|
|
|
|
|
* Save/restore needed during syscalls. Restore syscall arguments from
|
|
|
|
|
* the values already saved on stack during kernel_entry.
|
|
|
|
|
*/
|
|
|
|
|
ldp x0, x1, [sp]
|
|
|
|
|
ldp x2, x3, [sp, #S_X2]
|
|
|
|
|
ldp x4, x5, [sp, #S_X4]
|
|
|
|
|
ldp x6, x7, [sp, #S_X6]
|
|
|
|
|
.endif
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.macro ct_user_enter
|
|
|
|
|
#ifdef CONFIG_CONTEXT_TRACKING
|
|
|
|
|
bl context_tracking_user_enter
|
|
|
|
|
#endif
|
|
|
|
|
.endm
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* Bad Abort numbers
|
|
|
|
|
*-----------------
|
|
|
|
|
*/
|
|
|
|
|
#define BAD_SYNC 0
|
|
|
|
|
#define BAD_IRQ 1
|
|
|
|
|
#define BAD_FIQ 2
|
|
|
|
|
#define BAD_ERROR 3
|
|
|
|
|
|
|
|
|
|
.macro kernel_entry, el, regsize = 64
|
2014-09-29 19:26:41 +08:00
|
|
|
sub sp, sp, #S_FRAME_SIZE
|
2012-03-05 19:49:27 +08:00
|
|
|
.if \regsize == 32
|
|
|
|
|
mov w0, w0 // zero upper 32 bits of x0
|
|
|
|
|
.endif
|
2014-09-29 19:26:41 +08:00
|
|
|
stp x0, x1, [sp, #16 * 0]
|
|
|
|
|
stp x2, x3, [sp, #16 * 1]
|
|
|
|
|
stp x4, x5, [sp, #16 * 2]
|
|
|
|
|
stp x6, x7, [sp, #16 * 3]
|
|
|
|
|
stp x8, x9, [sp, #16 * 4]
|
|
|
|
|
stp x10, x11, [sp, #16 * 5]
|
|
|
|
|
stp x12, x13, [sp, #16 * 6]
|
|
|
|
|
stp x14, x15, [sp, #16 * 7]
|
|
|
|
|
stp x16, x17, [sp, #16 * 8]
|
|
|
|
|
stp x18, x19, [sp, #16 * 9]
|
|
|
|
|
stp x20, x21, [sp, #16 * 10]
|
|
|
|
|
stp x22, x23, [sp, #16 * 11]
|
|
|
|
|
stp x24, x25, [sp, #16 * 12]
|
|
|
|
|
stp x26, x27, [sp, #16 * 13]
|
|
|
|
|
stp x28, x29, [sp, #16 * 14]
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
.if \el == 0
|
|
|
|
|
mrs x21, sp_el0
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr_this_cpu tsk, __entry_task, x20 // Ensure MDSCR_EL1.SS is clear,
|
|
|
|
|
ldr x19, [tsk, #TSK_TI_FLAGS] // since we can unmask debug
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
disable_step_tsk x19, x20 // exceptions when scheduling.
|
2015-12-10 18:22:41 +08:00
|
|
|
|
|
|
|
|
mov x29, xzr // fp pointed to user-space
|
2012-03-05 19:49:27 +08:00
|
|
|
.else
|
|
|
|
|
add x21, sp, #S_FRAME_SIZE
|
2016-06-21 01:28:01 +08:00
|
|
|
get_thread_info tsk
|
|
|
|
|
/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x20, [tsk, #TSK_TI_ADDR_LIMIT]
|
2016-06-21 01:28:01 +08:00
|
|
|
str x20, [sp, #S_ORIG_ADDR_LIMIT]
|
|
|
|
|
mov x20, #TASK_SIZE_64
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
str x20, [tsk, #TSK_TI_ADDR_LIMIT]
|
2016-09-01 21:35:59 +08:00
|
|
|
/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
|
2016-06-21 01:28:01 +08:00
|
|
|
.endif /* \el == 0 */
|
2012-03-05 19:49:27 +08:00
|
|
|
mrs x22, elr_el1
|
|
|
|
|
mrs x23, spsr_el1
|
|
|
|
|
stp lr, x21, [sp, #S_LR]
|
2016-09-02 21:54:03 +08:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
|
|
|
|
|
/*
|
|
|
|
|
* Set the TTBR0 PAN bit in SPSR. When the exception is taken from
|
|
|
|
|
* EL0, there is no need to check the state of TTBR0_EL1 since
|
|
|
|
|
* accesses are always enabled.
|
|
|
|
|
* Note that the meaning of this bit differs from the ARMv8.1 PAN
|
|
|
|
|
* feature as all TTBR0_EL1 accesses are disabled, not just those to
|
|
|
|
|
* user mappings.
|
|
|
|
|
*/
|
|
|
|
|
alternative_if ARM64_HAS_PAN
|
|
|
|
|
b 1f // skip TTBR0 PAN
|
|
|
|
|
alternative_else_nop_endif
|
|
|
|
|
|
|
|
|
|
.if \el != 0
|
|
|
|
|
mrs x21, ttbr0_el1
|
|
|
|
|
tst x21, #0xffff << 48 // Check for the reserved ASID
|
|
|
|
|
orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR
|
|
|
|
|
b.eq 1f // TTBR0 access already disabled
|
|
|
|
|
and x23, x23, #~PSR_PAN_BIT // Clear the emulated PAN in the saved SPSR
|
|
|
|
|
.endif
|
|
|
|
|
|
|
|
|
|
__uaccess_ttbr0_disable x21
|
|
|
|
|
1:
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
stp x22, x23, [sp, #S_PC]
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Set syscallno to -1 by default (overridden later if real syscall).
|
|
|
|
|
*/
|
|
|
|
|
.if \el == 0
|
|
|
|
|
mvn x21, xzr
|
|
|
|
|
str x21, [sp, #S_SYSCALLNO]
|
|
|
|
|
.endif
|
|
|
|
|
|
2015-12-04 19:02:25 +08:00
|
|
|
/*
|
|
|
|
|
* Set sp_el0 to current thread_info.
|
|
|
|
|
*/
|
|
|
|
|
.if \el == 0
|
|
|
|
|
msr sp_el0, tsk
|
|
|
|
|
.endif
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* Registers that may be useful after this macro is invoked:
|
|
|
|
|
*
|
|
|
|
|
* x21 - aborted SP
|
|
|
|
|
* x22 - aborted PC
|
|
|
|
|
* x23 - aborted PSTATE
|
|
|
|
|
*/
|
|
|
|
|
.endm
|
|
|
|
|
|
2015-08-19 22:57:09 +08:00
|
|
|
.macro kernel_exit, el
|
2016-06-21 01:28:01 +08:00
|
|
|
.if \el != 0
|
|
|
|
|
/* Restore the task's original addr_limit. */
|
|
|
|
|
ldr x20, [sp, #S_ORIG_ADDR_LIMIT]
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
str x20, [tsk, #TSK_TI_ADDR_LIMIT]
|
2016-06-21 01:28:01 +08:00
|
|
|
|
|
|
|
|
/* No need to restore UAO, it will be restored from SPSR_EL1 */
|
|
|
|
|
.endif
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
|
|
|
|
|
.if \el == 0
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_enter
|
2016-09-02 21:54:03 +08:00
|
|
|
.endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
|
|
|
|
|
/*
|
|
|
|
|
* Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
|
|
|
|
|
* PAN bit checking.
|
|
|
|
|
*/
|
|
|
|
|
alternative_if ARM64_HAS_PAN
|
|
|
|
|
b 2f // skip TTBR0 PAN
|
|
|
|
|
alternative_else_nop_endif
|
|
|
|
|
|
|
|
|
|
.if \el != 0
|
|
|
|
|
tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
|
|
|
|
|
.endif
|
|
|
|
|
|
|
|
|
|
__uaccess_ttbr0_enable x0
|
|
|
|
|
|
|
|
|
|
.if \el == 0
|
|
|
|
|
/*
|
|
|
|
|
* Enable errata workarounds only if returning to user. The only
|
|
|
|
|
* workaround currently required for TTBR0_EL1 changes are for the
|
|
|
|
|
* Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
|
|
|
|
|
* corruption).
|
|
|
|
|
*/
|
|
|
|
|
post_ttbr0_update_workaround
|
|
|
|
|
.endif
|
|
|
|
|
1:
|
|
|
|
|
.if \el != 0
|
|
|
|
|
and x22, x22, #~PSR_PAN_BIT // ARMv8.0 CPUs do not understand this bit
|
|
|
|
|
.endif
|
|
|
|
|
2:
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
.if \el == 0
|
2012-03-05 19:49:27 +08:00
|
|
|
ldr x23, [sp, #S_SP] // load return stack pointer
|
2014-09-29 19:26:41 +08:00
|
|
|
msr sp_el0, x23
|
2015-03-24 03:07:02 +08:00
|
|
|
#ifdef CONFIG_ARM64_ERRATUM_845719
|
2016-09-07 18:07:09 +08:00
|
|
|
alternative_if ARM64_WORKAROUND_845719
|
2015-07-22 19:21:03 +08:00
|
|
|
tbz x22, #4, 1f
|
|
|
|
|
#ifdef CONFIG_PID_IN_CONTEXTIDR
|
|
|
|
|
mrs x29, contextidr_el1
|
|
|
|
|
msr contextidr_el1, x29
|
2015-03-24 03:07:02 +08:00
|
|
|
#else
|
2015-07-22 19:21:03 +08:00
|
|
|
msr contextidr_el1, xzr
|
2015-03-24 03:07:02 +08:00
|
|
|
#endif
|
2015-07-22 19:21:03 +08:00
|
|
|
1:
|
2016-09-07 18:07:09 +08:00
|
|
|
alternative_else_nop_endif
|
2015-03-24 03:07:02 +08:00
|
|
|
#endif
|
2012-03-05 19:49:27 +08:00
|
|
|
.endif
|
2016-09-02 21:54:03 +08:00
|
|
|
|
2014-09-29 19:26:41 +08:00
|
|
|
msr elr_el1, x21 // set up the return data
|
|
|
|
|
msr spsr_el1, x22
|
|
|
|
|
ldp x0, x1, [sp, #16 * 0]
|
|
|
|
|
ldp x2, x3, [sp, #16 * 1]
|
|
|
|
|
ldp x4, x5, [sp, #16 * 2]
|
|
|
|
|
ldp x6, x7, [sp, #16 * 3]
|
|
|
|
|
ldp x8, x9, [sp, #16 * 4]
|
|
|
|
|
ldp x10, x11, [sp, #16 * 5]
|
|
|
|
|
ldp x12, x13, [sp, #16 * 6]
|
|
|
|
|
ldp x14, x15, [sp, #16 * 7]
|
|
|
|
|
ldp x16, x17, [sp, #16 * 8]
|
|
|
|
|
ldp x18, x19, [sp, #16 * 9]
|
|
|
|
|
ldp x20, x21, [sp, #16 * 10]
|
|
|
|
|
ldp x22, x23, [sp, #16 * 11]
|
|
|
|
|
ldp x24, x25, [sp, #16 * 12]
|
|
|
|
|
ldp x26, x27, [sp, #16 * 13]
|
|
|
|
|
ldp x28, x29, [sp, #16 * 14]
|
|
|
|
|
ldr lr, [sp, #S_LR]
|
|
|
|
|
add sp, sp, #S_FRAME_SIZE // restore sp
|
2012-03-05 19:49:27 +08:00
|
|
|
eret // return to kernel
|
|
|
|
|
.endm
|
|
|
|
|
|
2015-12-15 19:21:25 +08:00
|
|
|
.macro irq_stack_entry
|
2015-12-04 19:02:27 +08:00
|
|
|
mov x19, sp // preserve the original sp
|
|
|
|
|
|
|
|
|
|
/*
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
* Compare sp with the base of the task stack.
|
|
|
|
|
* If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
|
|
|
|
|
* and should switch to the irq stack.
|
2015-12-04 19:02:27 +08:00
|
|
|
*/
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x25, [tsk, TSK_STACK]
|
|
|
|
|
eor x25, x25, x19
|
|
|
|
|
and x25, x25, #~(THREAD_SIZE - 1)
|
|
|
|
|
cbnz x25, 9998f
|
2015-12-04 19:02:27 +08:00
|
|
|
|
2016-11-04 04:23:12 +08:00
|
|
|
adr_this_cpu x25, irq_stack, x26
|
2015-12-04 19:02:27 +08:00
|
|
|
mov x26, #IRQ_STACK_START_SP
|
|
|
|
|
add x26, x25, x26
|
arm64: remove irq_count and do_softirq_own_stack()
sysrq_handle_reboot() re-enables interrupts while on the irq stack. The
irq_stack implementation wrongly assumed this would only ever happen
via the softirq path, allowing it to update irq_count late, in
do_softirq_own_stack().
This means if an irq occurs in sysrq_handle_reboot(), during
emergency_restart() the stack will be corrupted, as irq_count wasn't
updated.
Lose the optimisation, and instead of moving the adding/subtracting of
irq_count into irq_stack_entry/irq_stack_exit, remove it, and compare
sp_el0 (struct thread_info) with sp & ~(THREAD_SIZE - 1). This tells us
if we are on a task stack, if so, we can safely switch to the irq stack.
Finally, remove do_softirq_own_stack(), we don't need it anymore.
Reported-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
[will: use get_thread_info macro]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2015-12-19 00:01:47 +08:00
|
|
|
|
|
|
|
|
/* switch to the irq stack */
|
2015-12-04 19:02:27 +08:00
|
|
|
mov sp, x26
|
|
|
|
|
|
2015-12-15 19:21:25 +08:00
|
|
|
/*
|
|
|
|
|
* Add a dummy stack frame, this non-standard format is fixed up
|
|
|
|
|
* by unwind_frame()
|
|
|
|
|
*/
|
|
|
|
|
stp x29, x19, [sp, #-16]!
|
2015-12-04 19:02:27 +08:00
|
|
|
mov x29, sp
|
|
|
|
|
|
|
|
|
|
9998:
|
|
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* x19 should be preserved between irq_stack_entry and
|
|
|
|
|
* irq_stack_exit.
|
|
|
|
|
*/
|
|
|
|
|
.macro irq_stack_exit
|
|
|
|
|
mov sp, x19
|
|
|
|
|
.endm
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* These are the registers used in the syscall handler, and allow us to
|
|
|
|
|
* have in theory up to 7 arguments to a function - x0 to x6.
|
|
|
|
|
*
|
|
|
|
|
* x7 is reserved for the system call number in 32-bit mode.
|
|
|
|
|
*/
|
|
|
|
|
sc_nr .req x25 // number of system calls
|
|
|
|
|
scno .req x26 // syscall number
|
|
|
|
|
stbl .req x27 // syscall table pointer
|
|
|
|
|
tsk .req x28 // current thread_info
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Interrupt handling.
|
|
|
|
|
*/
|
|
|
|
|
.macro irq_handler
|
2015-12-04 19:02:27 +08:00
|
|
|
ldr_l x1, handle_arch_irq
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, sp
|
2015-12-15 19:21:25 +08:00
|
|
|
irq_stack_entry
|
2012-03-05 19:49:27 +08:00
|
|
|
blr x1
|
2015-12-04 19:02:27 +08:00
|
|
|
irq_stack_exit
|
2012-03-05 19:49:27 +08:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
.text
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Exception vectors.
|
|
|
|
|
*/
|
2016-07-09 00:35:50 +08:00
|
|
|
.pushsection ".entry.text", "ax"
|
2012-03-05 19:49:27 +08:00
|
|
|
|
|
|
|
|
.align 11
|
|
|
|
|
ENTRY(vectors)
|
|
|
|
|
ventry el1_sync_invalid // Synchronous EL1t
|
|
|
|
|
ventry el1_irq_invalid // IRQ EL1t
|
|
|
|
|
ventry el1_fiq_invalid // FIQ EL1t
|
|
|
|
|
ventry el1_error_invalid // Error EL1t
|
|
|
|
|
|
|
|
|
|
ventry el1_sync // Synchronous EL1h
|
|
|
|
|
ventry el1_irq // IRQ EL1h
|
|
|
|
|
ventry el1_fiq_invalid // FIQ EL1h
|
|
|
|
|
ventry el1_error_invalid // Error EL1h
|
|
|
|
|
|
|
|
|
|
ventry el0_sync // Synchronous 64-bit EL0
|
|
|
|
|
ventry el0_irq // IRQ 64-bit EL0
|
|
|
|
|
ventry el0_fiq_invalid // FIQ 64-bit EL0
|
|
|
|
|
ventry el0_error_invalid // Error 64-bit EL0
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
ventry el0_sync_compat // Synchronous 32-bit EL0
|
|
|
|
|
ventry el0_irq_compat // IRQ 32-bit EL0
|
|
|
|
|
ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
|
|
|
|
|
ventry el0_error_invalid_compat // Error 32-bit EL0
|
|
|
|
|
#else
|
|
|
|
|
ventry el0_sync_invalid // Synchronous 32-bit EL0
|
|
|
|
|
ventry el0_irq_invalid // IRQ 32-bit EL0
|
|
|
|
|
ventry el0_fiq_invalid // FIQ 32-bit EL0
|
|
|
|
|
ventry el0_error_invalid // Error 32-bit EL0
|
|
|
|
|
#endif
|
|
|
|
|
END(vectors)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Invalid mode handlers
|
|
|
|
|
*/
|
|
|
|
|
.macro inv_entry, el, reason, regsize = 64
|
2016-03-18 17:58:09 +08:00
|
|
|
kernel_entry \el, \regsize
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, sp
|
|
|
|
|
mov x1, #\reason
|
|
|
|
|
mrs x2, esr_el1
|
arm64: consistently use bl for C exception entry
In most cases, our exception entry assembly branches to C handlers with
a BL instruction, but in cases where we do not expect to return, we use
B instead.
While this is correct today, it means that backtraces for fatal
exceptions miss the entry assembly (as the LR is stale at the point we
call C code), while non-fatal exceptions have the entry assembly in the
LR. In subsequent patches, we will need the LR to be set in these cases
in order to backtrace reliably.
This patch updates these sites to use a BL, ensuring consistency, and
preparing for backtrace rework. An ASM_BUG() is added after each of
these new BLs, which both catches unexpected returns, and ensures that
the LR value doesn't point to another function label.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
2017-07-26 18:14:53 +08:00
|
|
|
bl bad_mode
|
|
|
|
|
ASM_BUG()
|
2012-03-05 19:49:27 +08:00
|
|
|
.endm
|
|
|
|
|
|
|
|
|
|
el0_sync_invalid:
|
|
|
|
|
inv_entry 0, BAD_SYNC
|
|
|
|
|
ENDPROC(el0_sync_invalid)
|
|
|
|
|
|
|
|
|
|
el0_irq_invalid:
|
|
|
|
|
inv_entry 0, BAD_IRQ
|
|
|
|
|
ENDPROC(el0_irq_invalid)
|
|
|
|
|
|
|
|
|
|
el0_fiq_invalid:
|
|
|
|
|
inv_entry 0, BAD_FIQ
|
|
|
|
|
ENDPROC(el0_fiq_invalid)
|
|
|
|
|
|
|
|
|
|
el0_error_invalid:
|
|
|
|
|
inv_entry 0, BAD_ERROR
|
|
|
|
|
ENDPROC(el0_error_invalid)
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
el0_fiq_invalid_compat:
|
|
|
|
|
inv_entry 0, BAD_FIQ, 32
|
|
|
|
|
ENDPROC(el0_fiq_invalid_compat)
|
|
|
|
|
|
|
|
|
|
el0_error_invalid_compat:
|
|
|
|
|
inv_entry 0, BAD_ERROR, 32
|
|
|
|
|
ENDPROC(el0_error_invalid_compat)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
el1_sync_invalid:
|
|
|
|
|
inv_entry 1, BAD_SYNC
|
|
|
|
|
ENDPROC(el1_sync_invalid)
|
|
|
|
|
|
|
|
|
|
el1_irq_invalid:
|
|
|
|
|
inv_entry 1, BAD_IRQ
|
|
|
|
|
ENDPROC(el1_irq_invalid)
|
|
|
|
|
|
|
|
|
|
el1_fiq_invalid:
|
|
|
|
|
inv_entry 1, BAD_FIQ
|
|
|
|
|
ENDPROC(el1_fiq_invalid)
|
|
|
|
|
|
|
|
|
|
el1_error_invalid:
|
|
|
|
|
inv_entry 1, BAD_ERROR
|
|
|
|
|
ENDPROC(el1_error_invalid)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* EL1 mode handlers.
|
|
|
|
|
*/
|
|
|
|
|
.align 6
|
|
|
|
|
el1_sync:
|
|
|
|
|
kernel_entry 1
|
|
|
|
|
mrs x1, esr_el1 // read the syndrome register
|
2014-11-24 20:31:40 +08:00
|
|
|
lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
|
|
|
|
|
cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el1_da
|
2016-08-10 09:25:26 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_IABT_CUR // instruction abort in EL1
|
|
|
|
|
b.eq el1_ia
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el1_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el1_sp_pc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el1_sp_pc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el1_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
|
2012-03-05 19:49:27 +08:00
|
|
|
b.ge el1_dbg
|
|
|
|
|
b el1_inv
|
2016-08-10 09:25:26 +08:00
|
|
|
|
|
|
|
|
el1_ia:
|
|
|
|
|
/*
|
|
|
|
|
* Fall through to the Data abort case
|
|
|
|
|
*/
|
2012-03-05 19:49:27 +08:00
|
|
|
el1_da:
|
|
|
|
|
/*
|
|
|
|
|
* Data abort handling
|
|
|
|
|
*/
|
arm64: entry: improve data abort handling of tagged pointers
When handling a data abort from EL0, we currently zero the top byte of
the faulting address, as we assume the address is a TTBR0 address, which
may contain a non-zero address tag. However, the address may be a TTBR1
address, in which case we should not zero the top byte. This patch fixes
that. The effect is that the full TTBR1 address is passed to the task's
signal handler (or printed out in the kernel log).
When handling a data abort from EL1, we leave the faulting address
intact, as we assume it's either a TTBR1 address or a TTBR0 address with
tag 0x00. This is true as far as I'm aware, we don't seem to access a
tagged TTBR0 address anywhere in the kernel. Regardless, it's easy to
forget about address tags, and code added in the future may not always
remember to remove tags from addresses before accessing them. So add tag
handling to the EL1 data abort handler as well. This also makes it
consistent with the EL0 data abort handler.
Fixes: d50240a5f6ce ("arm64: mm: permit use of tagged pointers at EL0")
Cc: <stable@vger.kernel.org> # 3.12.x-
Reviewed-by: Dave Martin <Dave.Martin@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-05-03 23:37:47 +08:00
|
|
|
mrs x3, far_el1
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2012-03-05 19:49:27 +08:00
|
|
|
// re-enable interrupts if they were enabled in the aborted context
|
|
|
|
|
tbnz x23, #7, 1f // PSR_I_BIT
|
|
|
|
|
enable_irq
|
|
|
|
|
1:
|
arm64: entry: improve data abort handling of tagged pointers
When handling a data abort from EL0, we currently zero the top byte of
the faulting address, as we assume the address is a TTBR0 address, which
may contain a non-zero address tag. However, the address may be a TTBR1
address, in which case we should not zero the top byte. This patch fixes
that. The effect is that the full TTBR1 address is passed to the task's
signal handler (or printed out in the kernel log).
When handling a data abort from EL1, we leave the faulting address
intact, as we assume it's either a TTBR1 address or a TTBR0 address with
tag 0x00. This is true as far as I'm aware, we don't seem to access a
tagged TTBR0 address anywhere in the kernel. Regardless, it's easy to
forget about address tags, and code added in the future may not always
remember to remove tags from addresses before accessing them. So add tag
handling to the EL1 data abort handler as well. This also makes it
consistent with the EL0 data abort handler.
Fixes: d50240a5f6ce ("arm64: mm: permit use of tagged pointers at EL0")
Cc: <stable@vger.kernel.org> # 3.12.x-
Reviewed-by: Dave Martin <Dave.Martin@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-05-03 23:37:47 +08:00
|
|
|
clear_address_tag x0, x3
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x2, sp // struct pt_regs
|
|
|
|
|
bl do_mem_abort
|
|
|
|
|
|
|
|
|
|
// disable interrupts before pulling preserved data off the stack
|
|
|
|
|
disable_irq
|
|
|
|
|
kernel_exit 1
|
|
|
|
|
el1_sp_pc:
|
|
|
|
|
/*
|
|
|
|
|
* Stack or PC alignment exception handling
|
|
|
|
|
*/
|
|
|
|
|
mrs x0, far_el1
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x2, sp
|
arm64: consistently use bl for C exception entry
In most cases, our exception entry assembly branches to C handlers with
a BL instruction, but in cases where we do not expect to return, we use
B instead.
While this is correct today, it means that backtraces for fatal
exceptions miss the entry assembly (as the LR is stale at the point we
call C code), while non-fatal exceptions have the entry assembly in the
LR. In subsequent patches, we will need the LR to be set in these cases
in order to backtrace reliably.
This patch updates these sites to use a BL, ensuring consistency, and
preparing for backtrace rework. An ASM_BUG() is added after each of
these new BLs, which both catches unexpected returns, and ensures that
the LR value doesn't point to another function label.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
2017-07-26 18:14:53 +08:00
|
|
|
bl do_sp_pc_abort
|
|
|
|
|
ASM_BUG()
|
2012-03-05 19:49:27 +08:00
|
|
|
el1_undef:
|
|
|
|
|
/*
|
|
|
|
|
* Undefined instruction
|
|
|
|
|
*/
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, sp
|
arm64: consistently use bl for C exception entry
In most cases, our exception entry assembly branches to C handlers with
a BL instruction, but in cases where we do not expect to return, we use
B instead.
While this is correct today, it means that backtraces for fatal
exceptions miss the entry assembly (as the LR is stale at the point we
call C code), while non-fatal exceptions have the entry assembly in the
LR. In subsequent patches, we will need the LR to be set in these cases
in order to backtrace reliably.
This patch updates these sites to use a BL, ensuring consistency, and
preparing for backtrace rework. An ASM_BUG() is added after each of
these new BLs, which both catches unexpected returns, and ensures that
the LR value doesn't point to another function label.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
2017-07-26 18:14:53 +08:00
|
|
|
bl do_undefinstr
|
|
|
|
|
ASM_BUG()
|
2012-03-05 19:49:27 +08:00
|
|
|
el1_dbg:
|
|
|
|
|
/*
|
|
|
|
|
* Debug exception handling
|
|
|
|
|
*/
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
|
2013-12-04 13:50:20 +08:00
|
|
|
cinc x24, x24, eq // set bit '0'
|
2012-03-05 19:49:27 +08:00
|
|
|
tbz x24, #0, el1_inv // EL1 only
|
|
|
|
|
mrs x0, far_el1
|
|
|
|
|
mov x2, sp // struct pt_regs
|
|
|
|
|
bl do_debug_exception
|
|
|
|
|
kernel_exit 1
|
|
|
|
|
el1_inv:
|
|
|
|
|
// TODO: add support for undefined instructions in kernel mode
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, sp
|
2015-07-08 01:00:49 +08:00
|
|
|
mov x2, x1
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x1, #BAD_SYNC
|
arm64: consistently use bl for C exception entry
In most cases, our exception entry assembly branches to C handlers with
a BL instruction, but in cases where we do not expect to return, we use
B instead.
While this is correct today, it means that backtraces for fatal
exceptions miss the entry assembly (as the LR is stale at the point we
call C code), while non-fatal exceptions have the entry assembly in the
LR. In subsequent patches, we will need the LR to be set in these cases
in order to backtrace reliably.
This patch updates these sites to use a BL, ensuring consistency, and
preparing for backtrace rework. An ASM_BUG() is added after each of
these new BLs, which both catches unexpected returns, and ensures that
the LR value doesn't point to another function label.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
2017-07-26 18:14:53 +08:00
|
|
|
bl bad_mode
|
|
|
|
|
ASM_BUG()
|
2012-03-05 19:49:27 +08:00
|
|
|
ENDPROC(el1_sync)
|
|
|
|
|
|
|
|
|
|
.align 6
|
|
|
|
|
el1_irq:
|
|
|
|
|
kernel_entry 1
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2012-03-05 19:49:27 +08:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
|
bl trace_hardirqs_off
|
|
|
|
|
#endif
|
2013-11-13 01:11:53 +08:00
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
irq_handler
|
2013-11-13 01:11:53 +08:00
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
#ifdef CONFIG_PREEMPT
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr w24, [tsk, #TSK_TI_PREEMPT] // get preempt count
|
2013-11-05 04:14:58 +08:00
|
|
|
cbnz w24, 1f // preempt count != 0
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x0, [tsk, #TSK_TI_FLAGS] // get flags
|
2012-03-05 19:49:27 +08:00
|
|
|
tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
|
|
|
|
|
bl el1_preempt
|
|
|
|
|
1:
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
|
bl trace_hardirqs_on
|
|
|
|
|
#endif
|
|
|
|
|
kernel_exit 1
|
|
|
|
|
ENDPROC(el1_irq)
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
|
el1_preempt:
|
|
|
|
|
mov x24, lr
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
1: bl preempt_schedule_irq // irq en/disable is done inside
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x0, [tsk, #TSK_TI_FLAGS] // get new tasks TI_FLAGS
|
2012-03-05 19:49:27 +08:00
|
|
|
tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling?
|
|
|
|
|
ret x24
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* EL0 mode handlers.
|
|
|
|
|
*/
|
|
|
|
|
.align 6
|
|
|
|
|
el0_sync:
|
|
|
|
|
kernel_entry 0
|
|
|
|
|
mrs x25, esr_el1 // read the syndrome register
|
2014-11-24 20:31:40 +08:00
|
|
|
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
|
|
|
|
|
cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_svc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_da
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_ia
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_fpsimd_acc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_fpsimd_exc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
|
2016-06-29 01:07:32 +08:00
|
|
|
b.eq el0_sys
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_sp_pc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_sp_pc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.ge el0_dbg
|
|
|
|
|
b el0_inv
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
.align 6
|
|
|
|
|
el0_sync_compat:
|
|
|
|
|
kernel_entry 0, 32
|
|
|
|
|
mrs x25, esr_el1 // read the syndrome register
|
2014-11-24 20:31:40 +08:00
|
|
|
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
|
|
|
|
|
cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_svc_compat
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_da
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_ia
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_fpsimd_acc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_fpsimd_exc
|
2015-10-14 05:30:51 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
|
|
|
|
b.eq el0_sp_pc
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
|
2013-05-24 19:02:35 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
|
2013-05-24 19:02:35 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
|
2013-05-24 19:02:35 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
|
2013-05-24 19:02:35 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
|
2013-05-24 19:02:35 +08:00
|
|
|
b.eq el0_undef
|
2014-11-24 20:31:40 +08:00
|
|
|
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
|
2012-03-05 19:49:27 +08:00
|
|
|
b.ge el0_dbg
|
|
|
|
|
b el0_inv
|
|
|
|
|
el0_svc_compat:
|
|
|
|
|
/*
|
|
|
|
|
* AArch32 syscall handling
|
|
|
|
|
*/
|
2015-01-07 00:42:32 +08:00
|
|
|
adrp stbl, compat_sys_call_table // load compat syscall table pointer
|
2012-03-05 19:49:27 +08:00
|
|
|
uxtw scno, w7 // syscall number in w7 (r7)
|
|
|
|
|
mov sc_nr, #__NR_compat_syscalls
|
|
|
|
|
b el0_svc_naked
|
|
|
|
|
|
|
|
|
|
.align 6
|
|
|
|
|
el0_irq_compat:
|
|
|
|
|
kernel_entry 0, 32
|
|
|
|
|
b el0_irq_naked
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
el0_da:
|
|
|
|
|
/*
|
|
|
|
|
* Data abort handling
|
|
|
|
|
*/
|
2014-05-31 03:34:14 +08:00
|
|
|
mrs x26, far_el1
|
2012-03-05 19:49:27 +08:00
|
|
|
// enable interrupts before calling the main handler
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg_and_irq
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
arm64: entry: improve data abort handling of tagged pointers
When handling a data abort from EL0, we currently zero the top byte of
the faulting address, as we assume the address is a TTBR0 address, which
may contain a non-zero address tag. However, the address may be a TTBR1
address, in which case we should not zero the top byte. This patch fixes
that. The effect is that the full TTBR1 address is passed to the task's
signal handler (or printed out in the kernel log).
When handling a data abort from EL1, we leave the faulting address
intact, as we assume it's either a TTBR1 address or a TTBR0 address with
tag 0x00. This is true as far as I'm aware, we don't seem to access a
tagged TTBR0 address anywhere in the kernel. Regardless, it's easy to
forget about address tags, and code added in the future may not always
remember to remove tags from addresses before accessing them. So add tag
handling to the EL1 data abort handler as well. This also makes it
consistent with the EL0 data abort handler.
Fixes: d50240a5f6ce ("arm64: mm: permit use of tagged pointers at EL0")
Cc: <stable@vger.kernel.org> # 3.12.x-
Reviewed-by: Dave Martin <Dave.Martin@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-05-03 23:37:47 +08:00
|
|
|
clear_address_tag x0, x26
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x1, x25
|
|
|
|
|
mov x2, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_mem_abort
|
|
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_ia:
|
|
|
|
|
/*
|
|
|
|
|
* Instruction abort handling
|
|
|
|
|
*/
|
2014-05-31 03:34:14 +08:00
|
|
|
mrs x26, far_el1
|
2012-03-05 19:49:27 +08:00
|
|
|
// enable interrupts before calling the main handler
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg_and_irq
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
2014-05-31 03:34:14 +08:00
|
|
|
mov x0, x26
|
arm64: kill ESR_LNX_EXEC
Currently we treat ESR_EL1 bit 24 as software-defined for distinguishing
instruction aborts from data aborts, but this bit is architecturally
RES0 for instruction aborts, and could be allocated for an arbitrary
purpose in future. Additionally, we hard-code the value in entry.S
without the mnemonic, making the code difficult to understand.
Instead, remove ESR_LNX_EXEC, and distinguish aborts based on the esr,
which we already pass to the sole use of ESR_LNX_EXEC. A new helper,
is_el0_instruction_abort() is added to make the logic clear. Any
instruction aborts taken from EL1 will already have been handled by
bad_mode, so we need not handle that case in the helper.
For consistency, the existing permission_fault helper is renamed to
is_permission_fault, and the return type is changed to bool. There
should be no functional changes as the return value was a boolean
expression, and the result is only used in another boolean expression.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Dave P Martin <dave.martin@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-05-31 19:33:03 +08:00
|
|
|
mov x1, x25
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x2, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_mem_abort
|
|
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_fpsimd_acc:
|
|
|
|
|
/*
|
|
|
|
|
* Floating Point or Advanced SIMD access
|
|
|
|
|
*/
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, x25
|
|
|
|
|
mov x1, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_fpsimd_acc
|
|
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_fpsimd_exc:
|
|
|
|
|
/*
|
|
|
|
|
* Floating Point or Advanced SIMD exception
|
|
|
|
|
*/
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, x25
|
|
|
|
|
mov x1, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_fpsimd_exc
|
|
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_sp_pc:
|
|
|
|
|
/*
|
|
|
|
|
* Stack or PC alignment exception handling
|
|
|
|
|
*/
|
2014-05-31 03:34:14 +08:00
|
|
|
mrs x26, far_el1
|
2012-03-05 19:49:27 +08:00
|
|
|
// enable interrupts before calling the main handler
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg_and_irq
|
2015-06-15 23:40:27 +08:00
|
|
|
ct_user_exit
|
2014-05-31 03:34:14 +08:00
|
|
|
mov x0, x26
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x1, x25
|
|
|
|
|
mov x2, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_sp_pc_abort
|
|
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_undef:
|
|
|
|
|
/*
|
|
|
|
|
* Undefined instruction
|
|
|
|
|
*/
|
2013-08-22 18:47:37 +08:00
|
|
|
// enable interrupts before calling the main handler
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg_and_irq
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
mov x0, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_undefinstr
|
|
|
|
|
b ret_to_user
|
2016-06-29 01:07:32 +08:00
|
|
|
el0_sys:
|
|
|
|
|
/*
|
|
|
|
|
* System instructions, for trapped cache maintenance instructions
|
|
|
|
|
*/
|
|
|
|
|
enable_dbg_and_irq
|
|
|
|
|
ct_user_exit
|
|
|
|
|
mov x0, x25
|
|
|
|
|
mov x1, sp
|
|
|
|
|
bl do_sysinstr
|
|
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_dbg:
|
|
|
|
|
/*
|
|
|
|
|
* Debug exception handling
|
|
|
|
|
*/
|
|
|
|
|
tbnz x24, #0, el0_inv // EL0 only
|
|
|
|
|
mrs x0, far_el1
|
|
|
|
|
mov x1, x25
|
|
|
|
|
mov x2, sp
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
bl do_debug_exception
|
|
|
|
|
enable_dbg
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
el0_inv:
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
2012-03-05 19:49:27 +08:00
|
|
|
mov x0, sp
|
|
|
|
|
mov x1, #BAD_SYNC
|
2015-07-08 01:00:49 +08:00
|
|
|
mov x2, x25
|
arm64: avoid returning from bad_mode
Generally, taking an unexpected exception should be a fatal event, and
bad_mode is intended to cater for this. However, it should be possible
to contain unexpected synchronous exceptions from EL0 without bringing
the kernel down, by sending a SIGILL to the task.
We tried to apply this approach in commit 9955ac47f4ba1c95 ("arm64:
don't kill the kernel on a bad esr from el0"), by sending a signal for
any bad_mode call resulting from an EL0 exception.
However, this also applies to other unexpected exceptions, such as
SError and FIQ. The entry paths for these exceptions branch to bad_mode
without configuring the link register, and have no kernel_exit. Thus, if
we take one of these exceptions from EL0, bad_mode will eventually
return to the original user link register value.
This patch fixes this by introducing a new bad_el0_sync handler to cater
for the recoverable case, and restoring bad_mode to its original state,
whereby it calls panic() and never returns. The recoverable case
branches to bad_el0_sync with a bl, and returns to userspace via the
usual ret_to_user mechanism.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Fixes: 9955ac47f4ba1c95 ("arm64: don't kill the kernel on a bad esr from el0")
Reported-by: Mark Salter <msalter@redhat.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: stable@vger.kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-01-19 01:23:41 +08:00
|
|
|
bl bad_el0_sync
|
2014-09-29 18:44:01 +08:00
|
|
|
b ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
ENDPROC(el0_sync)
|
|
|
|
|
|
|
|
|
|
.align 6
|
|
|
|
|
el0_irq:
|
|
|
|
|
kernel_entry 0
|
|
|
|
|
el0_irq_naked:
|
|
|
|
|
enable_dbg
|
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
|
bl trace_hardirqs_off
|
|
|
|
|
#endif
|
2013-11-13 01:11:53 +08:00
|
|
|
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit
|
2012-03-05 19:49:27 +08:00
|
|
|
irq_handler
|
2013-11-13 01:11:53 +08:00
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
|
bl trace_hardirqs_on
|
|
|
|
|
#endif
|
|
|
|
|
b ret_to_user
|
|
|
|
|
ENDPROC(el0_irq)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This is the fast syscall return path. We do as little as possible here,
|
|
|
|
|
* and this includes saving x0 back into the kernel stack.
|
|
|
|
|
*/
|
|
|
|
|
ret_fast_syscall:
|
|
|
|
|
disable_irq // disable interrupts
|
2015-08-19 22:57:09 +08:00
|
|
|
str x0, [sp, #S_X0] // returned x0
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing
|
2015-06-06 05:28:03 +08:00
|
|
|
and x2, x1, #_TIF_SYSCALL_WORK
|
|
|
|
|
cbnz x2, ret_fast_syscall_trace
|
2012-03-05 19:49:27 +08:00
|
|
|
and x2, x1, #_TIF_WORK_MASK
|
2015-08-19 22:57:09 +08:00
|
|
|
cbnz x2, work_pending
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_step_tsk x1, x2
|
2015-08-19 22:57:09 +08:00
|
|
|
kernel_exit 0
|
2015-06-06 05:28:03 +08:00
|
|
|
ret_fast_syscall_trace:
|
|
|
|
|
enable_irq // enable interrupts
|
2015-08-19 22:57:09 +08:00
|
|
|
b __sys_trace_return_skipped // we already saved x0
|
2012-03-05 19:49:27 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Ok, we need to do extra processing, enter the slow path.
|
|
|
|
|
*/
|
|
|
|
|
work_pending:
|
|
|
|
|
mov x0, sp // 'regs'
|
|
|
|
|
bl do_notify_resume
|
2015-12-04 20:42:29 +08:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
2016-07-15 04:48:14 +08:00
|
|
|
bl trace_hardirqs_on // enabled while in userspace
|
2015-12-04 20:42:29 +08:00
|
|
|
#endif
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for single-step
|
2016-07-15 04:48:14 +08:00
|
|
|
b finish_ret_to_user
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* "slow" syscall return path.
|
|
|
|
|
*/
|
2012-09-10 23:11:46 +08:00
|
|
|
ret_to_user:
|
2012-03-05 19:49:27 +08:00
|
|
|
disable_irq // disable interrupts
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x1, [tsk, #TSK_TI_FLAGS]
|
2012-03-05 19:49:27 +08:00
|
|
|
and x2, x1, #_TIF_WORK_MASK
|
|
|
|
|
cbnz x2, work_pending
|
2016-07-15 04:48:14 +08:00
|
|
|
finish_ret_to_user:
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_step_tsk x1, x2
|
2015-08-19 22:57:09 +08:00
|
|
|
kernel_exit 0
|
2012-03-05 19:49:27 +08:00
|
|
|
ENDPROC(ret_to_user)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* SVC handler.
|
|
|
|
|
*/
|
|
|
|
|
.align 6
|
|
|
|
|
el0_svc:
|
|
|
|
|
adrp stbl, sys_call_table // load syscall table pointer
|
|
|
|
|
uxtw scno, w8 // syscall number in w8
|
|
|
|
|
mov sc_nr, #__NR_syscalls
|
|
|
|
|
el0_svc_naked: // compat entry point
|
|
|
|
|
stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
|
arm64: debug: avoid accessing mdscr_el1 on fault paths where possible
Since mdscr_el1 is part of the debug register group, it is highly likely
to be trapped by a hypervisor to prevent virtual machines from debugging
(buggering?) each other. Unfortunately, this absolutely destroys our
performance, since we access the register on many of our low-level
fault handling paths to keep track of the various debug state machines.
This patch removes our dependency on mdscr_el1 in the case that debugging
is not being used. More specifically we:
- Use TIF_SINGLESTEP to indicate that a task is stepping at EL0 and
avoid disabling step in the MDSCR when we don't need to.
MDSCR_EL1.SS handling is moved to kernel_entry, when trapping from
userspace.
- Ensure debug exceptions are re-enabled on *all* exception entry
paths, even the debug exception handling path (where we re-enable
exceptions after invoking the handler). Since we can now rely on
MDSCR_EL1.SS being cleared by the entry code, exception handlers can
usually enable debug immediately before enabling interrupts.
- Remove all debug exception unmasking from ret_to_user and
el1_preempt, since we will never get here with debug exceptions
masked.
This results in a slight change to kernel debug behaviour, where we now
step into interrupt handlers and data aborts from EL1 when debugging the
kernel, which is actually a useful thing to do. A side-effect of this is
that it *does* potentially prevent stepping off {break,watch}points when
there is a high-frequency interrupt source (e.g. a timer), so a debugger
would need to use either breakpoints or manually disable interrupts to
get around this issue.
With this patch applied, guest performance is restored under KVM when
debug register accesses are trapped (and we get a measurable performance
increase on the host on Cortex-A57 too).
Cc: Ian Campbell <ian.campbell@citrix.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 02:04:06 +08:00
|
|
|
enable_dbg_and_irq
|
2014-05-31 03:34:15 +08:00
|
|
|
ct_user_exit 1
|
2012-03-05 19:49:27 +08:00
|
|
|
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks
|
2014-04-30 17:51:29 +08:00
|
|
|
tst x16, #_TIF_SYSCALL_WORK
|
|
|
|
|
b.ne __sys_trace
|
2012-03-05 19:49:27 +08:00
|
|
|
cmp scno, sc_nr // check upper syscall limit
|
|
|
|
|
b.hs ni_sys
|
|
|
|
|
ldr x16, [stbl, scno, lsl #3] // address in the syscall table
|
2014-09-29 18:44:01 +08:00
|
|
|
blr x16 // call sys_* routine
|
|
|
|
|
b ret_fast_syscall
|
2012-03-05 19:49:27 +08:00
|
|
|
ni_sys:
|
|
|
|
|
mov x0, sp
|
2014-09-29 18:44:01 +08:00
|
|
|
bl do_ni_syscall
|
|
|
|
|
b ret_fast_syscall
|
2012-03-05 19:49:27 +08:00
|
|
|
ENDPROC(el0_svc)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This is the really slow path. We're going to be doing context
|
|
|
|
|
* switches, and waiting for our parent to respond.
|
|
|
|
|
*/
|
|
|
|
|
__sys_trace:
|
2014-11-28 13:26:35 +08:00
|
|
|
mov w0, #-1 // set default errno for
|
|
|
|
|
cmp scno, x0 // user-issued syscall(-1)
|
|
|
|
|
b.ne 1f
|
|
|
|
|
mov x0, #-ENOSYS
|
|
|
|
|
str x0, [sp, #S_X0]
|
|
|
|
|
1: mov x0, sp
|
2014-04-30 17:51:30 +08:00
|
|
|
bl syscall_trace_enter
|
2014-11-28 13:26:35 +08:00
|
|
|
cmp w0, #-1 // skip the syscall?
|
|
|
|
|
b.eq __sys_trace_return_skipped
|
2012-03-05 19:49:27 +08:00
|
|
|
uxtw scno, w0 // syscall number (possibly new)
|
|
|
|
|
mov x1, sp // pointer to regs
|
|
|
|
|
cmp scno, sc_nr // check upper syscall limit
|
2014-09-29 18:44:01 +08:00
|
|
|
b.hs __ni_sys_trace
|
2012-03-05 19:49:27 +08:00
|
|
|
ldp x0, x1, [sp] // restore the syscall args
|
|
|
|
|
ldp x2, x3, [sp, #S_X2]
|
|
|
|
|
ldp x4, x5, [sp, #S_X4]
|
|
|
|
|
ldp x6, x7, [sp, #S_X6]
|
|
|
|
|
ldr x16, [stbl, scno, lsl #3] // address in the syscall table
|
2014-09-29 18:44:01 +08:00
|
|
|
blr x16 // call sys_* routine
|
2012-03-05 19:49:27 +08:00
|
|
|
|
|
|
|
|
__sys_trace_return:
|
2014-11-28 13:26:35 +08:00
|
|
|
str x0, [sp, #S_X0] // save returned x0
|
|
|
|
|
__sys_trace_return_skipped:
|
2014-04-30 17:51:30 +08:00
|
|
|
mov x0, sp
|
|
|
|
|
bl syscall_trace_exit
|
2012-03-05 19:49:27 +08:00
|
|
|
b ret_to_user
|
|
|
|
|
|
2014-09-29 18:44:01 +08:00
|
|
|
__ni_sys_trace:
|
|
|
|
|
mov x0, sp
|
|
|
|
|
bl do_ni_syscall
|
|
|
|
|
b __sys_trace_return
|
|
|
|
|
|
2016-07-09 00:35:50 +08:00
|
|
|
.popsection // .entry.text
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* Special system call wrappers.
|
|
|
|
|
*/
|
|
|
|
|
ENTRY(sys_rt_sigreturn_wrapper)
|
|
|
|
|
mov x0, sp
|
|
|
|
|
b sys_rt_sigreturn
|
|
|
|
|
ENDPROC(sys_rt_sigreturn_wrapper)
|
2017-07-26 23:05:20 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Register switch for AArch64. The callee-saved registers need to be saved
|
|
|
|
|
* and restored. On entry:
|
|
|
|
|
* x0 = previous task_struct (must be preserved across the switch)
|
|
|
|
|
* x1 = next task_struct
|
|
|
|
|
* Previous and next are guaranteed not to be the same.
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
ENTRY(cpu_switch_to)
|
|
|
|
|
mov x10, #THREAD_CPU_CONTEXT
|
|
|
|
|
add x8, x0, x10
|
|
|
|
|
mov x9, sp
|
|
|
|
|
stp x19, x20, [x8], #16 // store callee-saved registers
|
|
|
|
|
stp x21, x22, [x8], #16
|
|
|
|
|
stp x23, x24, [x8], #16
|
|
|
|
|
stp x25, x26, [x8], #16
|
|
|
|
|
stp x27, x28, [x8], #16
|
|
|
|
|
stp x29, x9, [x8], #16
|
|
|
|
|
str lr, [x8]
|
|
|
|
|
add x8, x1, x10
|
|
|
|
|
ldp x19, x20, [x8], #16 // restore callee-saved registers
|
|
|
|
|
ldp x21, x22, [x8], #16
|
|
|
|
|
ldp x23, x24, [x8], #16
|
|
|
|
|
ldp x25, x26, [x8], #16
|
|
|
|
|
ldp x27, x28, [x8], #16
|
|
|
|
|
ldp x29, x9, [x8], #16
|
|
|
|
|
ldr lr, [x8]
|
|
|
|
|
mov sp, x9
|
|
|
|
|
msr sp_el0, x1
|
|
|
|
|
ret
|
|
|
|
|
ENDPROC(cpu_switch_to)
|
|
|
|
|
NOKPROBE(cpu_switch_to)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This is how we return from a fork.
|
|
|
|
|
*/
|
|
|
|
|
ENTRY(ret_from_fork)
|
|
|
|
|
bl schedule_tail
|
|
|
|
|
cbz x19, 1f // not a kernel thread
|
|
|
|
|
mov x0, x20
|
|
|
|
|
blr x19
|
|
|
|
|
1: get_thread_info tsk
|
|
|
|
|
b ret_to_user
|
|
|
|
|
ENDPROC(ret_from_fork)
|
|
|
|
|
NOKPROBE(ret_from_fork)
|