2019-06-03 13:44:50 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* Based on arch/arm/mm/fault.c
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1995 Linus Torvalds
|
|
|
|
|
* Copyright (C) 1995-2004 Russell King
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
2020-12-10 16:07:19 +08:00
|
|
|
* Copyright (C) 2020 Ampere Computing LLC
|
2012-03-05 19:49:27 +08:00
|
|
|
*/
|
|
|
|
|
|
2019-01-30 02:48:50 +08:00
|
|
|
#include <linux/acpi.h>
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
#include <linux/bitfield.h>
|
2016-09-20 05:38:55 +08:00
|
|
|
#include <linux/extable.h>
|
2021-02-26 09:19:03 +08:00
|
|
|
#include <linux/kfence.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <linux/signal.h>
|
|
|
|
|
#include <linux/mm.h>
|
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
|
#include <linux/init.h>
|
2020-12-23 04:02:13 +08:00
|
|
|
#include <linux/kasan.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <linux/kprobes.h>
|
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
|
#include <linux/page-flags.h>
|
2017-02-09 01:51:30 +08:00
|
|
|
#include <linux/sched/signal.h>
|
2017-02-09 01:51:35 +08:00
|
|
|
#include <linux/sched/debug.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <linux/highmem.h>
|
|
|
|
|
#include <linux/perf_event.h>
|
2016-10-18 18:27:47 +08:00
|
|
|
#include <linux/preempt.h>
|
2017-06-09 01:25:27 +08:00
|
|
|
#include <linux/hugetlb.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2019-01-30 02:48:50 +08:00
|
|
|
#include <asm/acpi.h>
|
2016-10-18 18:27:47 +08:00
|
|
|
#include <asm/bug.h>
|
2017-06-26 21:27:36 +08:00
|
|
|
#include <asm/cmpxchg.h>
|
2015-07-23 02:05:54 +08:00
|
|
|
#include <asm/cpufeature.h>
|
arm64: efi: Recover from synchronous exceptions occurring in firmware
Unlike x86, which has machinery to deal with page faults that occur
during the execution of EFI runtime services, arm64 has nothing like
that, and a synchronous exception raised by firmware code brings down
the whole system.
With more EFI based systems appearing that were not built to run Linux
(such as the Windows-on-ARM laptops based on Qualcomm SOCs), as well as
the introduction of PRM (platform specific firmware routines that are
callable just like EFI runtime services), we are more likely to run into
issues of this sort, and it is much more likely that we can identify and
work around such issues if they don't bring down the system entirely.
Since we already use a EFI runtime services call wrapper in assembler,
we can quite easily add some code that captures the execution state at
the point where the call is made, allowing us to revert to this state
and proceed execution if the call triggered a synchronous exception.
Given that the kernel and the firmware don't share any data structures
that could end up in an indeterminate state, we can happily continue
running, as long as we mark the EFI runtime services as unavailable from
that point on.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
2022-10-28 22:39:14 +08:00
|
|
|
#include <asm/efi.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/exception.h>
|
2018-08-28 23:51:15 +08:00
|
|
|
#include <asm/daifflags.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/debug-monitors.h>
|
2014-04-07 06:04:12 +08:00
|
|
|
#include <asm/esr.h>
|
2019-10-26 00:42:10 +08:00
|
|
|
#include <asm/kprobes.h>
|
2020-12-23 04:01:35 +08:00
|
|
|
#include <asm/mte.h>
|
2020-12-10 16:07:19 +08:00
|
|
|
#include <asm/patching.h>
|
2019-10-26 00:42:16 +08:00
|
|
|
#include <asm/processor.h>
|
2015-07-23 02:05:54 +08:00
|
|
|
#include <asm/sysreg.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
#include <asm/system_misc.h>
|
|
|
|
|
#include <asm/tlbflush.h>
|
2018-02-20 22:53:22 +08:00
|
|
|
#include <asm/traps.h>
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2017-04-04 13:51:01 +08:00
|
|
|
struct fault_info {
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
int (*fn)(unsigned long far, unsigned long esr,
|
2017-04-04 13:51:01 +08:00
|
|
|
struct pt_regs *regs);
|
|
|
|
|
int sig;
|
|
|
|
|
int code;
|
|
|
|
|
const char *name;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct fault_info fault_info[];
|
2018-09-22 23:39:54 +08:00
|
|
|
static struct fault_info debug_fault_info[];
|
2017-04-04 13:51:01 +08:00
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static inline const struct fault_info *esr_to_fault_info(unsigned long esr)
|
2017-04-04 13:51:01 +08:00
|
|
|
{
|
2018-09-22 23:39:52 +08:00
|
|
|
return fault_info + (esr & ESR_ELx_FSC);
|
2017-04-04 13:51:01 +08:00
|
|
|
}
|
2012-10-24 23:34:02 +08:00
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static inline const struct fault_info *esr_to_debug_fault_info(unsigned long esr)
|
2018-09-22 23:39:54 +08:00
|
|
|
{
|
|
|
|
|
return debug_fault_info + DBG_ESR_EVT(esr);
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void data_abort_decode(unsigned long esr)
|
2017-08-04 16:31:42 +08:00
|
|
|
{
|
2023-05-11 14:05:15 +08:00
|
|
|
unsigned long iss2 = ESR_ELx_ISS2(esr);
|
|
|
|
|
|
2017-08-04 16:31:42 +08:00
|
|
|
pr_alert("Data abort info:\n");
|
|
|
|
|
|
|
|
|
|
if (esr & ESR_ELx_ISV) {
|
|
|
|
|
pr_alert(" Access size = %u byte(s)\n",
|
|
|
|
|
1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
|
|
|
|
|
pr_alert(" SSE = %lu, SRT = %lu\n",
|
|
|
|
|
(esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
|
|
|
|
|
(esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
|
|
|
|
|
pr_alert(" SF = %lu, AR = %lu\n",
|
|
|
|
|
(esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
|
|
|
|
|
(esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
|
|
|
|
|
} else {
|
2023-05-11 14:05:15 +08:00
|
|
|
pr_alert(" ISV = 0, ISS = 0x%08lx, ISS2 = 0x%08lx\n",
|
|
|
|
|
esr & ESR_ELx_ISS_MASK, iss2);
|
2017-08-04 16:31:42 +08:00
|
|
|
}
|
|
|
|
|
|
2023-05-11 14:05:15 +08:00
|
|
|
pr_alert(" CM = %lu, WnR = %lu, TnD = %lu, TagAccess = %lu\n",
|
2017-08-04 16:31:42 +08:00
|
|
|
(esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
|
2023-05-11 14:05:15 +08:00
|
|
|
(esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT,
|
|
|
|
|
(iss2 & ESR_ELx_TnD) >> ESR_ELx_TnD_SHIFT,
|
|
|
|
|
(iss2 & ESR_ELx_TagAccess) >> ESR_ELx_TagAccess_SHIFT);
|
|
|
|
|
|
|
|
|
|
pr_alert(" GCS = %ld, Overlay = %lu, DirtyBit = %lu, Xs = %llu\n",
|
|
|
|
|
(iss2 & ESR_ELx_GCS) >> ESR_ELx_GCS_SHIFT,
|
|
|
|
|
(iss2 & ESR_ELx_Overlay) >> ESR_ELx_Overlay_SHIFT,
|
|
|
|
|
(iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT,
|
|
|
|
|
(iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT);
|
2017-08-04 16:31:42 +08:00
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void mem_abort_decode(unsigned long esr)
|
2017-08-04 16:31:42 +08:00
|
|
|
{
|
|
|
|
|
pr_alert("Mem abort info:\n");
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
pr_alert(" ESR = 0x%016lx\n", esr);
|
arm64: mm: print hexadecimal EC value in mem_abort_decode()
This change prints the hexadecimal EC value in mem_abort_decode(),
which makes it easier to lookup the corresponding EC in
the ARM Architecture Reference Manual.
The commit 1f9b8936f36f ("arm64: Decode information from ESR upon mem
faults") prints useful information when memory abort occurs. It would
be easier to lookup "0x25" instead of "DABT" in the document. Then we
can check the corresponding ISS.
For example:
Current info Document
EC Exception class
"CP15 MCR/MRC" 0x3 "MCR or MRC access to CP15a..."
"ASIMD" 0x7 "Access to SIMD or floating-point..."
"DABT (current EL)" 0x25 "Data Abort taken without..."
...
Before:
Unable to handle kernel paging request at virtual address 000000000000c000
Mem abort info:
ESR = 0x96000046
Exception class = DABT (current EL), IL = 32 bits
SET = 0, FnV = 0
EA = 0, S1PTW = 0
Data abort info:
ISV = 0, ISS = 0x00000046
CM = 0, WnR = 1
After:
Unable to handle kernel paging request at virtual address 000000000000c000
Mem abort info:
ESR = 0x96000046
EC = 0x25: DABT (current EL), IL = 32 bits
SET = 0, FnV = 0
EA = 0, S1PTW = 0
Data abort info:
ISV = 0, ISS = 0x00000046
CM = 0, WnR = 1
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: James Morse <james.morse@arm.com>
Acked-by: Mark Rutland <Mark.rutland@arm.com>
Signed-off-by: Miles Chen <miles.chen@mediatek.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-07 08:33:36 +08:00
|
|
|
pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
|
|
|
|
|
ESR_ELx_EC(esr), esr_get_class_string(esr),
|
2017-08-04 16:31:42 +08:00
|
|
|
(esr & ESR_ELx_IL) ? 32 : 16);
|
|
|
|
|
pr_alert(" SET = %lu, FnV = %lu\n",
|
|
|
|
|
(esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
|
|
|
|
|
(esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
|
|
|
|
|
pr_alert(" EA = %lu, S1PTW = %lu\n",
|
|
|
|
|
(esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
|
|
|
|
|
(esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
pr_alert(" FSC = 0x%02lx: %s\n", (esr & ESR_ELx_FSC),
|
2021-06-08 20:37:42 +08:00
|
|
|
esr_to_fault_info(esr)->name);
|
2017-08-04 16:31:42 +08:00
|
|
|
|
|
|
|
|
if (esr_is_data_abort(esr))
|
|
|
|
|
data_abort_decode(esr);
|
|
|
|
|
}
|
|
|
|
|
|
arm64: mm: avoid virt_to_phys(init_mm.pgd)
If we take an unhandled fault in the kernel, we call show_pte() to dump
the {PGDP,PGD,PUD,PMD,PTE} values for the corresponding page table walk,
where the PGDP value is virt_to_phys(mm->pgd).
The boot-time and runtime kernel page tables, init_pg_dir and
swapper_pg_dir respectively, are kernel symbols. Thus, it is not valid
to call virt_to_phys() on either of these, though we'll do so if we take
a fault on a TTBR1 address.
When CONFIG_DEBUG_VIRTUAL is not selected, virt_to_phys() will silently
fix this up. However, when CONFIG_DEBUG_VIRTUAL is selected, this
results in splats as below. Depending on when these occur, they can
happen to suppress information needed to debug the original unhandled
fault, such as the backtrace:
| Unable to handle kernel paging request at virtual address ffff7fffec73cf0f
| Mem abort info:
| ESR = 0x96000004
| EC = 0x25: DABT (current EL), IL = 32 bits
| SET = 0, FnV = 0
| EA = 0, S1PTW = 0
| Data abort info:
| ISV = 0, ISS = 0x00000004
| CM = 0, WnR = 0
| ------------[ cut here ]------------
| virt_to_phys used for non-linear address: 00000000102c9dbe (swapper_pg_dir+0x0/0x1000)
| WARNING: CPU: 1 PID: 7558 at arch/arm64/mm/physaddr.c:15 __virt_to_phys+0xe0/0x170 arch/arm64/mm/physaddr.c:12
| Kernel panic - not syncing: panic_on_warn set ...
| SMP: stopping secondary CPUs
| Dumping ftrace buffer:
| (ftrace buffer empty)
| Kernel Offset: disabled
| CPU features: 0x0002,23000438
| Memory Limit: none
| Rebooting in 1 seconds..
We can avoid this by ensuring that we call __pa_symbol() for
init_mm.pgd, as this will always be a kernel symbol. As the dumped
{PGD,PUD,PMD,PTE} values are the raw values from the relevant entries we
don't need to handle these specially.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-10-03 17:49:32 +08:00
|
|
|
static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
|
|
|
|
|
{
|
|
|
|
|
/* Either init_pg_dir or swapper_pg_dir */
|
|
|
|
|
if (mm == &init_mm)
|
|
|
|
|
return __pa_symbol(mm->pgd);
|
|
|
|
|
|
|
|
|
|
return (unsigned long)virt_to_phys(mm->pgd);
|
|
|
|
|
}
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
2017-06-09 23:35:52 +08:00
|
|
|
* Dump out the page tables associated with 'addr' in the currently active mm.
|
2012-03-05 19:49:27 +08:00
|
|
|
*/
|
2019-04-03 20:36:54 +08:00
|
|
|
static void show_pte(unsigned long addr)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2017-06-09 23:35:52 +08:00
|
|
|
struct mm_struct *mm;
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp;
|
|
|
|
|
pgd_t pgd;
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2018-12-28 16:30:27 +08:00
|
|
|
if (is_ttbr0_addr(addr)) {
|
2017-06-09 23:35:52 +08:00
|
|
|
/* TTBR0 */
|
|
|
|
|
mm = current->active_mm;
|
|
|
|
|
if (mm == &init_mm) {
|
|
|
|
|
pr_alert("[%016lx] user address but active_mm is swapper\n",
|
|
|
|
|
addr);
|
|
|
|
|
return;
|
|
|
|
|
}
|
2018-12-28 16:30:27 +08:00
|
|
|
} else if (is_ttbr1_addr(addr)) {
|
2017-06-09 23:35:52 +08:00
|
|
|
/* TTBR1 */
|
2012-03-05 19:49:27 +08:00
|
|
|
mm = &init_mm;
|
2017-06-09 23:35:52 +08:00
|
|
|
} else {
|
|
|
|
|
pr_alert("[%016lx] address between user and kernel address ranges\n",
|
|
|
|
|
addr);
|
|
|
|
|
return;
|
|
|
|
|
}
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2019-08-07 23:55:18 +08:00
|
|
|
pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n",
|
2017-05-15 22:23:58 +08:00
|
|
|
mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
|
arm64: mm: avoid virt_to_phys(init_mm.pgd)
If we take an unhandled fault in the kernel, we call show_pte() to dump
the {PGDP,PGD,PUD,PMD,PTE} values for the corresponding page table walk,
where the PGDP value is virt_to_phys(mm->pgd).
The boot-time and runtime kernel page tables, init_pg_dir and
swapper_pg_dir respectively, are kernel symbols. Thus, it is not valid
to call virt_to_phys() on either of these, though we'll do so if we take
a fault on a TTBR1 address.
When CONFIG_DEBUG_VIRTUAL is not selected, virt_to_phys() will silently
fix this up. However, when CONFIG_DEBUG_VIRTUAL is selected, this
results in splats as below. Depending on when these occur, they can
happen to suppress information needed to debug the original unhandled
fault, such as the backtrace:
| Unable to handle kernel paging request at virtual address ffff7fffec73cf0f
| Mem abort info:
| ESR = 0x96000004
| EC = 0x25: DABT (current EL), IL = 32 bits
| SET = 0, FnV = 0
| EA = 0, S1PTW = 0
| Data abort info:
| ISV = 0, ISS = 0x00000004
| CM = 0, WnR = 0
| ------------[ cut here ]------------
| virt_to_phys used for non-linear address: 00000000102c9dbe (swapper_pg_dir+0x0/0x1000)
| WARNING: CPU: 1 PID: 7558 at arch/arm64/mm/physaddr.c:15 __virt_to_phys+0xe0/0x170 arch/arm64/mm/physaddr.c:12
| Kernel panic - not syncing: panic_on_warn set ...
| SMP: stopping secondary CPUs
| Dumping ftrace buffer:
| (ftrace buffer empty)
| Kernel Offset: disabled
| CPU features: 0x0002,23000438
| Memory Limit: none
| Rebooting in 1 seconds..
We can avoid this by ensuring that we call __pa_symbol() for
init_mm.pgd, as this will always be a kernel symbol. As the dumped
{PGD,PUD,PMD,PTE} values are the raw values from the relevant entries we
don't need to handle these specially.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-10-03 17:49:32 +08:00
|
|
|
vabits_actual, mm_to_pgd_phys(mm));
|
2018-02-15 19:14:56 +08:00
|
|
|
pgdp = pgd_offset(mm, addr);
|
|
|
|
|
pgd = READ_ONCE(*pgdp);
|
|
|
|
|
pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
|
2012-03-05 19:49:27 +08:00
|
|
|
|
|
|
|
|
do {
|
2020-06-05 07:46:23 +08:00
|
|
|
p4d_t *p4dp, p4d;
|
2018-02-15 19:14:56 +08:00
|
|
|
pud_t *pudp, pud;
|
|
|
|
|
pmd_t *pmdp, pmd;
|
|
|
|
|
pte_t *ptep, pte;
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (pgd_none(pgd) || pgd_bad(pgd))
|
2012-03-05 19:49:27 +08:00
|
|
|
break;
|
|
|
|
|
|
2020-06-05 07:46:23 +08:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
|
p4d = READ_ONCE(*p4dp);
|
|
|
|
|
pr_cont(", p4d=%016llx", p4d_val(p4d));
|
|
|
|
|
if (p4d_none(p4d) || p4d_bad(p4d))
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
pudp = pud_offset(p4dp, addr);
|
2018-02-15 19:14:56 +08:00
|
|
|
pud = READ_ONCE(*pudp);
|
|
|
|
|
pr_cont(", pud=%016llx", pud_val(pud));
|
|
|
|
|
if (pud_none(pud) || pud_bad(pud))
|
2012-03-05 19:49:27 +08:00
|
|
|
break;
|
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
|
pmd = READ_ONCE(*pmdp);
|
|
|
|
|
pr_cont(", pmd=%016llx", pmd_val(pmd));
|
|
|
|
|
if (pmd_none(pmd) || pmd_bad(pmd))
|
2012-03-05 19:49:27 +08:00
|
|
|
break;
|
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = pte_offset_map(pmdp, addr);
|
2023-06-09 03:11:59 +08:00
|
|
|
if (!ptep)
|
|
|
|
|
break;
|
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pte = READ_ONCE(*ptep);
|
|
|
|
|
pr_cont(", pte=%016llx", pte_val(pte));
|
|
|
|
|
pte_unmap(ptep);
|
2012-03-05 19:49:27 +08:00
|
|
|
} while(0);
|
|
|
|
|
|
2017-01-03 22:27:26 +08:00
|
|
|
pr_cont("\n");
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
2016-04-13 23:01:22 +08:00
|
|
|
/*
|
|
|
|
|
* This function sets the access flags (dirty, accessed), as well as write
|
|
|
|
|
* permission, and only to a more permissive setting.
|
|
|
|
|
*
|
|
|
|
|
* It needs to cope with hardware update of the accessed/dirty state by other
|
|
|
|
|
* agents in the system and can safely skip the __sync_icache_dcache() call as,
|
|
|
|
|
* like set_pte_at(), the PTE is never changed from no-exec to exec here.
|
|
|
|
|
*
|
|
|
|
|
* Returns whether or not the PTE actually changed.
|
|
|
|
|
*/
|
|
|
|
|
int ptep_set_access_flags(struct vm_area_struct *vma,
|
|
|
|
|
unsigned long address, pte_t *ptep,
|
|
|
|
|
pte_t entry, int dirty)
|
|
|
|
|
{
|
2017-06-26 21:27:36 +08:00
|
|
|
pteval_t old_pteval, pteval;
|
2018-02-15 19:14:56 +08:00
|
|
|
pte_t pte = READ_ONCE(*ptep);
|
2016-04-13 23:01:22 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (pte_same(pte, entry))
|
2016-04-13 23:01:22 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/* only preserve the access flags and write permission */
|
2017-07-05 02:04:18 +08:00
|
|
|
pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
|
2016-04-13 23:01:22 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Setting the flags must be done atomically to avoid racing with the
|
2017-07-25 21:53:03 +08:00
|
|
|
* hardware update of the access/dirty state. The PTE_RDONLY bit must
|
|
|
|
|
* be set to the most permissive (lowest value) of *ptep and entry
|
|
|
|
|
* (calculated as: a & b == ~(~a | ~b)).
|
2016-04-13 23:01:22 +08:00
|
|
|
*/
|
2017-07-25 21:53:03 +08:00
|
|
|
pte_val(entry) ^= PTE_RDONLY;
|
2018-02-15 19:14:56 +08:00
|
|
|
pteval = pte_val(pte);
|
2017-06-26 21:27:36 +08:00
|
|
|
do {
|
|
|
|
|
old_pteval = pteval;
|
|
|
|
|
pteval ^= PTE_RDONLY;
|
|
|
|
|
pteval |= pte_val(entry);
|
|
|
|
|
pteval ^= PTE_RDONLY;
|
|
|
|
|
pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
|
|
|
|
|
} while (pteval != old_pteval);
|
2016-04-13 23:01:22 +08:00
|
|
|
|
2020-09-30 20:20:40 +08:00
|
|
|
/* Invalidate a stale read-only entry */
|
|
|
|
|
if (dirty)
|
|
|
|
|
flush_tlb_page(vma, address);
|
2016-04-13 23:01:22 +08:00
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static bool is_el1_instruction_abort(unsigned long esr)
|
2016-08-10 09:25:26 +08:00
|
|
|
{
|
|
|
|
|
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static bool is_el1_data_abort(unsigned long esr)
|
2021-06-03 20:02:39 +08:00
|
|
|
{
|
|
|
|
|
return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static inline bool is_el1_permission_fault(unsigned long addr, unsigned long esr,
|
2018-09-22 23:39:53 +08:00
|
|
|
struct pt_regs *regs)
|
2017-04-06 03:18:31 +08:00
|
|
|
{
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
unsigned long fsc_type = esr & ESR_ELx_FSC_TYPE;
|
2017-04-06 03:18:31 +08:00
|
|
|
|
2021-06-03 20:02:39 +08:00
|
|
|
if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
|
2017-04-06 03:18:31 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
if (fsc_type == ESR_ELx_FSC_PERM)
|
|
|
|
|
return true;
|
|
|
|
|
|
2018-12-28 16:30:27 +08:00
|
|
|
if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
|
2017-04-06 03:18:31 +08:00
|
|
|
return fsc_type == ESR_ELx_FSC_FAULT &&
|
|
|
|
|
(regs->pstate & PSR_PAN_BIT);
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
unsigned long esr,
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
u64 par, dfsc;
|
|
|
|
|
|
2021-06-03 20:02:39 +08:00
|
|
|
if (!is_el1_data_abort(esr) ||
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
(esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
asm volatile("at s1e1r, %0" :: "r" (addr));
|
|
|
|
|
isb();
|
2020-10-29 02:28:39 +08:00
|
|
|
par = read_sysreg_par();
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
2019-10-16 19:03:04 +08:00
|
|
|
/*
|
|
|
|
|
* If we now have a valid translation, treat the translation fault as
|
|
|
|
|
* spurious.
|
|
|
|
|
*/
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
if (!(par & SYS_PAR_EL1_F))
|
2019-10-16 19:03:04 +08:00
|
|
|
return true;
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If we got a different type of fault from the AT instruction,
|
|
|
|
|
* treat the translation fault as spurious.
|
|
|
|
|
*/
|
2019-10-04 21:58:47 +08:00
|
|
|
dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
return (dfsc & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT;
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-21 21:14:51 +08:00
|
|
|
static void die_kernel_fault(const char *msg, unsigned long addr,
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
unsigned long esr, struct pt_regs *regs)
|
2018-05-21 21:14:51 +08:00
|
|
|
{
|
|
|
|
|
bust_spinlocks(1);
|
|
|
|
|
|
|
|
|
|
pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
|
|
|
|
|
addr);
|
|
|
|
|
|
arm64: mm: log potential KASAN shadow alias
When the kernel is built with KASAN_GENERIC or KASAN_SW_TAGS, shadow
memory is allocated and mapped for all legitimate kernel addresses, and
prior to a regular memory access instrumentation will read from the
corresponding shadow address.
Due to the way memory addresses are converted to shadow addresses, bogus
pointers (e.g. NULL) can generate shadow addresses out of the bounds of
allocated shadow memory. For example, with KASAN_GENERIC and 48-bit VAs,
NULL would have a shadow address of dfff800000000000, which falls
between the TTBR ranges.
To make such cases easier to debug, this patch makes die_kernel_fault()
dump the real memory address range for any potential KASAN shadow access
using kasan_non_canonical_hook(), which results in fault information as
below when KASAN is enabled:
| Unable to handle kernel paging request at virtual address dfff800000000017
| KASAN: null-ptr-deref in range [0x00000000000000b8-0x00000000000000bf]
| Mem abort info:
| ESR = 0x96000004
| EC = 0x25: DABT (current EL), IL = 32 bits
| SET = 0, FnV = 0
| EA = 0, S1PTW = 0
| FSC = 0x04: level 0 translation fault
| Data abort info:
| ISV = 0, ISS = 0x00000004
| CM = 0, WnR = 0
| [dfff800000000017] address between user and kernel address ranges
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Will Deacon <will@kernel.org>
Tested-by: Andrey Konovalov <andreyknvl@gmail.com>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211207183226.834557-3-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-12-08 02:32:26 +08:00
|
|
|
kasan_non_canonical_hook(addr);
|
|
|
|
|
|
2018-05-21 21:14:51 +08:00
|
|
|
mem_abort_decode(esr);
|
|
|
|
|
|
|
|
|
|
show_pte(addr);
|
|
|
|
|
die("Oops", regs, esr);
|
|
|
|
|
bust_spinlocks(0);
|
2021-06-29 03:52:01 +08:00
|
|
|
make_task_dead(SIGKILL);
|
2018-05-21 21:14:51 +08:00
|
|
|
}
|
|
|
|
|
|
2020-12-23 04:02:13 +08:00
|
|
|
#ifdef CONFIG_KASAN_HW_TAGS
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void report_tag_fault(unsigned long addr, unsigned long esr,
|
2020-12-23 04:01:35 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
2020-12-23 04:02:13 +08:00
|
|
|
/*
|
|
|
|
|
* SAS bits aren't set for all faults reported in EL1, so we can't
|
|
|
|
|
* find out access size.
|
|
|
|
|
*/
|
2021-07-14 22:38:43 +08:00
|
|
|
bool is_write = !!(esr & ESR_ELx_WNR);
|
kasan: use internal prototypes matching gcc-13 builtins
gcc-13 warns about function definitions for builtin interfaces that have a
different prototype, e.g.:
In file included from kasan_test.c:31:
kasan.h:574:6: error: conflicting types for built-in function '__asan_register_globals'; expected 'void(void *, long int)' [-Werror=builtin-declaration-mismatch]
574 | void __asan_register_globals(struct kasan_global *globals, size_t size);
kasan.h:577:6: error: conflicting types for built-in function '__asan_alloca_poison'; expected 'void(void *, long int)' [-Werror=builtin-declaration-mismatch]
577 | void __asan_alloca_poison(unsigned long addr, size_t size);
kasan.h:580:6: error: conflicting types for built-in function '__asan_load1'; expected 'void(void *)' [-Werror=builtin-declaration-mismatch]
580 | void __asan_load1(unsigned long addr);
kasan.h:581:6: error: conflicting types for built-in function '__asan_store1'; expected 'void(void *)' [-Werror=builtin-declaration-mismatch]
581 | void __asan_store1(unsigned long addr);
kasan.h:643:6: error: conflicting types for built-in function '__hwasan_tag_memory'; expected 'void(void *, unsigned char, long int)' [-Werror=builtin-declaration-mismatch]
643 | void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size);
The two problems are:
- Addresses are passes as 'unsigned long' in the kernel, but gcc-13
expects a 'void *'.
- sizes meant to use a signed ssize_t rather than size_t.
Change all the prototypes to match these. Using 'void *' consistently for
addresses gets rid of a couple of type casts, so push that down to the
leaf functions where possible.
This now passes all randconfig builds on arm, arm64 and x86, but I have
not tested it on the other architectures that support kasan, since they
tend to fail randconfig builds in other ways. This might fail if any of
the 32-bit architectures expect a 'long' instead of 'int' for the size
argument.
The __asan_allocas_unpoison() function prototype is somewhat weird, since
it uses a pointer for 'stack_top' and an size_t for 'stack_bottom'. This
looks like it is meant to be 'addr' and 'size' like the others, but the
implementation clearly treats them as 'top' and 'bottom'.
Link: https://lkml.kernel.org/r/20230509145735.9263-2-arnd@kernel.org
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-05-09 22:57:21 +08:00
|
|
|
kasan_report((void *)addr, 0, is_write, regs->pc);
|
2020-12-23 04:01:35 +08:00
|
|
|
}
|
2020-12-23 04:02:13 +08:00
|
|
|
#else
|
|
|
|
|
/* Tag faults aren't enabled without CONFIG_KASAN_HW_TAGS. */
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static inline void report_tag_fault(unsigned long addr, unsigned long esr,
|
2020-12-23 04:02:13 +08:00
|
|
|
struct pt_regs *regs) { }
|
|
|
|
|
#endif
|
2020-12-23 04:01:35 +08:00
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void do_tag_recovery(unsigned long addr, unsigned long esr,
|
2020-12-23 04:01:35 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
|
2021-02-25 04:05:26 +08:00
|
|
|
report_tag_fault(addr, esr, regs);
|
2020-12-23 04:01:35 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Disable MTE Tag Checking on the local CPU for the current EL.
|
|
|
|
|
* It will be done lazily on the other CPUs when they will hit a
|
|
|
|
|
* tag fault.
|
|
|
|
|
*/
|
2022-05-04 01:02:24 +08:00
|
|
|
sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK,
|
|
|
|
|
SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE));
|
2020-12-23 04:01:35 +08:00
|
|
|
isb();
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static bool is_el1_mte_sync_tag_check_fault(unsigned long esr)
|
2020-12-23 04:01:35 +08:00
|
|
|
{
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
unsigned long fsc = esr & ESR_ELx_FSC;
|
2020-12-23 04:01:35 +08:00
|
|
|
|
2021-06-03 20:02:39 +08:00
|
|
|
if (!is_el1_data_abort(esr))
|
2020-12-23 04:01:35 +08:00
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
if (fsc == ESR_ELx_FSC_MTE)
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
2022-11-14 18:44:11 +08:00
|
|
|
static bool is_translation_fault(unsigned long esr)
|
|
|
|
|
{
|
|
|
|
|
return (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_FAULT;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void __do_kernel_fault(unsigned long addr, unsigned long esr,
|
2017-06-09 23:35:52 +08:00
|
|
|
struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2017-04-06 03:18:31 +08:00
|
|
|
const char *msg;
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* Are we prepared to handle this kernel fault?
|
2016-08-10 09:25:26 +08:00
|
|
|
* We are almost certainly not prepared to handle instruction faults.
|
2012-03-05 19:49:27 +08:00
|
|
|
*/
|
2016-08-10 09:25:26 +08:00
|
|
|
if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
|
2012-03-05 19:49:27 +08:00
|
|
|
return;
|
|
|
|
|
|
arm64: mm: Ignore spurious translation faults taken from the kernel
Thanks to address translation being performed out of order with respect to
loads and stores, it is possible for a CPU to take a translation fault when
accessing a page that was mapped by a different CPU.
For example, in the case that one CPU maps a page and then sets a flag to
tell another CPU:
CPU 0
-----
MOV X0, <valid pte>
STR X0, [Xptep] // Store new PTE to page table
DSB ISHST
ISB
MOV X1, #1
STR X1, [Xflag] // Set the flag
CPU 1
-----
loop: LDAR X0, [Xflag] // Poll flag with Acquire semantics
CBZ X0, loop
LDR X1, [X2] // Translates using the new PTE
then the final load on CPU 1 can raise a translation fault because the
translation can be performed speculatively before the read of the flag and
marked as "faulting" by the CPU. This isn't quite as bad as it sounds
since, in reality, code such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
*ptr = vmalloc(size); if (*ptr)
spin_unlock(&lock); foo = **ptr;
spin_unlock(&lock);
will not trigger the fault because there is an address dependency on CPU 1
which prevents the speculative translation. However, more exotic code where
the virtual address is known ahead of time, such as:
CPU 0 CPU 1
----- -----
spin_lock(&lock); spin_lock(&lock);
set_fixmap(0, paddr, prot); if (mapped)
mapped = true; foo = *fix_to_virt(0);
spin_unlock(&lock); spin_unlock(&lock);
could fault. This can be avoided by any of:
* Introducing broadcast TLB maintenance on the map path
* Adding a DSB;ISB sequence after checking a flag which indicates
that a virtual address is now mapped
* Handling the spurious fault
Given that we have never observed a problem due to this under Linux and
future revisions of the architecture are being tightened so that
translation table walks are effectively ordered in the same way as explicit
memory accesses, we no longer treat spurious kernel faults as fatal if an
AT instruction indicates that the access does not trigger a translation
fault.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2019-08-23 00:22:14 +08:00
|
|
|
if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
|
|
|
|
|
"Ignoring spurious kernel translation fault at virtual address %016lx\n", addr))
|
|
|
|
|
return;
|
|
|
|
|
|
2020-12-23 04:01:35 +08:00
|
|
|
if (is_el1_mte_sync_tag_check_fault(esr)) {
|
|
|
|
|
do_tag_recovery(addr, esr, regs);
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-22 23:39:53 +08:00
|
|
|
if (is_el1_permission_fault(addr, esr, regs)) {
|
2017-04-06 03:18:31 +08:00
|
|
|
if (esr & ESR_ELx_WNR)
|
|
|
|
|
msg = "write to read-only memory";
|
2019-10-29 20:41:31 +08:00
|
|
|
else if (is_el1_instruction_abort(esr))
|
|
|
|
|
msg = "execute from non-executable memory";
|
2017-04-06 03:18:31 +08:00
|
|
|
else
|
|
|
|
|
msg = "read from unreadable memory";
|
|
|
|
|
} else if (addr < PAGE_SIZE) {
|
|
|
|
|
msg = "NULL pointer dereference";
|
|
|
|
|
} else {
|
2022-11-14 18:44:11 +08:00
|
|
|
if (is_translation_fault(esr) &&
|
|
|
|
|
kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
|
2021-02-26 09:19:03 +08:00
|
|
|
return;
|
|
|
|
|
|
2017-04-06 03:18:31 +08:00
|
|
|
msg = "paging request";
|
|
|
|
|
}
|
|
|
|
|
|
arm64: efi: Recover from synchronous exceptions occurring in firmware
Unlike x86, which has machinery to deal with page faults that occur
during the execution of EFI runtime services, arm64 has nothing like
that, and a synchronous exception raised by firmware code brings down
the whole system.
With more EFI based systems appearing that were not built to run Linux
(such as the Windows-on-ARM laptops based on Qualcomm SOCs), as well as
the introduction of PRM (platform specific firmware routines that are
callable just like EFI runtime services), we are more likely to run into
issues of this sort, and it is much more likely that we can identify and
work around such issues if they don't bring down the system entirely.
Since we already use a EFI runtime services call wrapper in assembler,
we can quite easily add some code that captures the execution state at
the point where the call is made, allowing us to revert to this state
and proceed execution if the call triggered a synchronous exception.
Given that the kernel and the firmware don't share any data structures
that could end up in an indeterminate state, we can happily continue
running, as long as we mark the EFI runtime services as unavailable from
that point on.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
2022-10-28 22:39:14 +08:00
|
|
|
if (efi_runtime_fixup_exception(regs, msg))
|
|
|
|
|
return;
|
|
|
|
|
|
2018-05-21 21:14:51 +08:00
|
|
|
die_kernel_fault(msg, addr, esr, regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void set_thread_esr(unsigned long address, unsigned long esr)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2018-09-22 15:37:55 +08:00
|
|
|
current->thread.fault_address = address;
|
2018-05-23 00:11:20 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If the faulting address is in the kernel, we must sanitize the ESR.
|
|
|
|
|
* From userspace's point of view, kernel-only mappings don't exist
|
|
|
|
|
* at all, so we report them as level 0 translation faults.
|
|
|
|
|
* (This is not quite the way that "no mapping there at all" behaves:
|
|
|
|
|
* an alignment fault not caused by the memory type would take
|
|
|
|
|
* precedence over translation fault for a real access to empty
|
|
|
|
|
* space. Unfortunately we can't easily distinguish "alignment fault
|
|
|
|
|
* not caused by memory type" from "alignment fault caused by memory
|
|
|
|
|
* type", so we ignore this wrinkle and just return the translation
|
|
|
|
|
* fault.)
|
|
|
|
|
*/
|
2018-12-28 16:30:27 +08:00
|
|
|
if (!is_ttbr0_addr(current->thread.fault_address)) {
|
2018-05-23 00:11:20 +08:00
|
|
|
switch (ESR_ELx_EC(esr)) {
|
|
|
|
|
case ESR_ELx_EC_DABT_LOW:
|
|
|
|
|
/*
|
|
|
|
|
* These bits provide only information about the
|
|
|
|
|
* faulting instruction, which userspace knows already.
|
|
|
|
|
* We explicitly clear bits which are architecturally
|
|
|
|
|
* RES0 in case they are given meanings in future.
|
|
|
|
|
* We always report the ESR as if the fault was taken
|
|
|
|
|
* to EL1 and so ISV and the bits in ISS[23:14] are
|
|
|
|
|
* clear. (In fact it always will be a fault to EL1.)
|
|
|
|
|
*/
|
|
|
|
|
esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
|
|
|
|
|
ESR_ELx_CM | ESR_ELx_WNR;
|
|
|
|
|
esr |= ESR_ELx_FSC_FAULT;
|
|
|
|
|
break;
|
|
|
|
|
case ESR_ELx_EC_IABT_LOW:
|
|
|
|
|
/*
|
|
|
|
|
* Claim a level 0 translation fault.
|
|
|
|
|
* All other bits are architecturally RES0 for faults
|
|
|
|
|
* reported with that DFSC value, so we clear them.
|
|
|
|
|
*/
|
|
|
|
|
esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
|
|
|
|
|
esr |= ESR_ELx_FSC_FAULT;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
/*
|
|
|
|
|
* This should never happen (entry.S only brings us
|
|
|
|
|
* into this code for insn and data aborts from a lower
|
|
|
|
|
* exception level). Fail safe by not providing an ESR
|
|
|
|
|
* context record at all.
|
|
|
|
|
*/
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
WARN(1, "ESR 0x%lx is not DABT or IABT from EL0\n", esr);
|
2018-05-23 00:11:20 +08:00
|
|
|
esr = 0;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-02-20 22:53:22 +08:00
|
|
|
current->thread.fault_code = esr;
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static void do_bad_area(unsigned long far, unsigned long esr,
|
2020-11-21 04:33:46 +08:00
|
|
|
struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2020-11-21 04:33:46 +08:00
|
|
|
unsigned long addr = untagged_addr(far);
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* If we are in kernel mode at this point, we have no context to
|
|
|
|
|
* handle this fault with.
|
|
|
|
|
*/
|
2017-04-04 13:51:01 +08:00
|
|
|
if (user_mode(regs)) {
|
2018-02-20 22:53:22 +08:00
|
|
|
const struct fault_info *inf = esr_to_fault_info(esr);
|
2018-04-18 04:26:37 +08:00
|
|
|
|
2018-09-22 16:05:41 +08:00
|
|
|
set_thread_esr(addr, esr);
|
2020-11-21 04:33:46 +08:00
|
|
|
arm64_force_sig_fault(inf->sig, inf->code, far, inf->name);
|
2018-02-20 22:53:22 +08:00
|
|
|
} else {
|
2017-06-09 23:35:52 +08:00
|
|
|
__do_kernel_fault(addr, esr, regs);
|
2018-02-20 22:53:22 +08:00
|
|
|
}
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
2023-05-02 23:19:06 +08:00
|
|
|
#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000)
|
|
|
|
|
#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000)
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2023-06-16 08:11:44 +08:00
|
|
|
static vm_fault_t __do_page_fault(struct mm_struct *mm,
|
|
|
|
|
struct vm_area_struct *vma, unsigned long addr,
|
2020-08-12 09:37:57 +08:00
|
|
|
unsigned int mm_flags, unsigned long vm_flags,
|
|
|
|
|
struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* Ok, we have a good vm_area for this memory access, so we can handle
|
|
|
|
|
* it.
|
2013-07-19 22:37:12 +08:00
|
|
|
* Check that the permissions on the VMA allow for the fault which
|
2016-08-12 01:44:50 +08:00
|
|
|
* occurred.
|
2013-07-19 22:37:12 +08:00
|
|
|
*/
|
2019-06-07 17:13:06 +08:00
|
|
|
if (!(vma->vm_flags & vm_flags))
|
|
|
|
|
return VM_FAULT_BADACCESS;
|
2021-06-14 20:27:01 +08:00
|
|
|
return handle_mm_fault(vma, addr, mm_flags, regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static bool is_el0_instruction_abort(unsigned long esr)
|
arm64: kill ESR_LNX_EXEC
Currently we treat ESR_EL1 bit 24 as software-defined for distinguishing
instruction aborts from data aborts, but this bit is architecturally
RES0 for instruction aborts, and could be allocated for an arbitrary
purpose in future. Additionally, we hard-code the value in entry.S
without the mnemonic, making the code difficult to understand.
Instead, remove ESR_LNX_EXEC, and distinguish aborts based on the esr,
which we already pass to the sole use of ESR_LNX_EXEC. A new helper,
is_el0_instruction_abort() is added to make the logic clear. Any
instruction aborts taken from EL1 will already have been handled by
bad_mode, so we need not handle that case in the helper.
For consistency, the existing permission_fault helper is renamed to
is_permission_fault, and the return type is changed to bool. There
should be no functional changes as the return value was a boolean
expression, and the result is only used in another boolean expression.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Dave P Martin <dave.martin@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-05-31 19:33:03 +08:00
|
|
|
{
|
|
|
|
|
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
|
|
|
|
|
}
|
|
|
|
|
|
2019-06-07 17:13:05 +08:00
|
|
|
/*
|
|
|
|
|
* Note: not valid for EL1 DC IVAC, but we never use that such that it
|
|
|
|
|
* should fault. EL0 cannot issue DC IVAC (undef).
|
|
|
|
|
*/
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static bool is_write_abort(unsigned long esr)
|
2019-06-07 17:13:05 +08:00
|
|
|
{
|
|
|
|
|
return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static int __kprobes do_page_fault(unsigned long far, unsigned long esr,
|
2012-03-05 19:49:27 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
2018-09-22 16:16:42 +08:00
|
|
|
const struct fault_info *inf;
|
2019-06-03 14:41:23 +08:00
|
|
|
struct mm_struct *mm = current->mm;
|
2020-08-12 09:37:57 +08:00
|
|
|
vm_fault_t fault;
|
2021-03-13 01:38:10 +08:00
|
|
|
unsigned long vm_flags;
|
2020-04-02 12:08:37 +08:00
|
|
|
unsigned int mm_flags = FAULT_FLAG_DEFAULT;
|
2020-11-21 04:33:46 +08:00
|
|
|
unsigned long addr = untagged_addr(far);
|
2023-02-28 01:36:29 +08:00
|
|
|
struct vm_area_struct *vma;
|
2013-07-19 22:37:12 +08:00
|
|
|
|
2019-07-17 07:28:00 +08:00
|
|
|
if (kprobe_page_fault(regs, esr))
|
arm64: Kprobes with single stepping support
Add support for basic kernel probes(kprobes) and jump probes
(jprobes) for ARM64.
Kprobes utilizes software breakpoint and single step debug
exceptions supported on ARM v8.
A software breakpoint is placed at the probe address to trap the
kernel execution into the kprobe handler.
ARM v8 supports enabling single stepping before the break exception
return (ERET), with next PC in exception return address (ELR_EL1). The
kprobe handler prepares an executable memory slot for out-of-line
execution with a copy of the original instruction being probed, and
enables single stepping. The PC is set to the out-of-line slot address
before the ERET. With this scheme, the instruction is executed with the
exact same register context except for the PC (and DAIF) registers.
Debug mask (PSTATE.D) is enabled only when single stepping a recursive
kprobe, e.g.: during kprobes reenter so that probed instruction can be
single stepped within the kprobe handler -exception- context.
The recursion depth of kprobe is always 2, i.e. upon probe re-entry,
any further re-entry is prevented by not calling handlers and the case
counted as a missed kprobe).
Single stepping from the x-o-l slot has a drawback for PC-relative accesses
like branching and symbolic literals access as the offset from the new PC
(slot address) may not be ensured to fit in the immediate value of
the opcode. Such instructions need simulation, so reject
probing them.
Instructions generating exceptions or cpu mode change are rejected
for probing.
Exclusive load/store instructions are rejected too. Additionally, the
code is checked to see if it is inside an exclusive load/store sequence
(code from Pratyush).
System instructions are mostly enabled for stepping, except MSR/MRS
accesses to "DAIF" flags in PSTATE, which are not safe for
probing.
This also changes arch/arm64/include/asm/ptrace.h to use
include/asm-generic/ptrace.h.
Thanks to Steve Capper and Pratyush Anand for several suggested
Changes.
Signed-off-by: Sandeepa Prabhu <sandeepa.s.prabhu@gmail.com>
Signed-off-by: David A. Long <dave.long@linaro.org>
Signed-off-by: Pratyush Anand <panand@redhat.com>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-09 00:35:48 +08:00
|
|
|
return 0;
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
|
|
|
|
* If we're in an interrupt or have no user context, we must not take
|
|
|
|
|
* the fault.
|
|
|
|
|
*/
|
2015-05-11 23:52:11 +08:00
|
|
|
if (faulthandler_disabled() || !mm)
|
2012-03-05 19:49:27 +08:00
|
|
|
goto no_context;
|
|
|
|
|
|
2013-09-13 06:13:39 +08:00
|
|
|
if (user_mode(regs))
|
|
|
|
|
mm_flags |= FAULT_FLAG_USER;
|
|
|
|
|
|
2021-03-13 01:38:10 +08:00
|
|
|
/*
|
|
|
|
|
* vm_flags tells us what bits we must have in vma->vm_flags
|
|
|
|
|
* for the fault to be benign, __do_page_fault() would check
|
|
|
|
|
* vma->vm_flags & vm_flags and returns an error if the
|
|
|
|
|
* intersection is empty
|
|
|
|
|
*/
|
arm64: kill ESR_LNX_EXEC
Currently we treat ESR_EL1 bit 24 as software-defined for distinguishing
instruction aborts from data aborts, but this bit is architecturally
RES0 for instruction aborts, and could be allocated for an arbitrary
purpose in future. Additionally, we hard-code the value in entry.S
without the mnemonic, making the code difficult to understand.
Instead, remove ESR_LNX_EXEC, and distinguish aborts based on the esr,
which we already pass to the sole use of ESR_LNX_EXEC. A new helper,
is_el0_instruction_abort() is added to make the logic clear. Any
instruction aborts taken from EL1 will already have been handled by
bad_mode, so we need not handle that case in the helper.
For consistency, the existing permission_fault helper is renamed to
is_permission_fault, and the return type is changed to bool. There
should be no functional changes as the return value was a boolean
expression, and the result is only used in another boolean expression.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Dave P Martin <dave.martin@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-05-31 19:33:03 +08:00
|
|
|
if (is_el0_instruction_abort(esr)) {
|
2021-03-13 01:38:10 +08:00
|
|
|
/* It was exec fault */
|
2013-09-13 06:13:39 +08:00
|
|
|
vm_flags = VM_EXEC;
|
2019-05-05 12:15:12 +08:00
|
|
|
mm_flags |= FAULT_FLAG_INSTRUCTION;
|
2019-06-07 17:13:05 +08:00
|
|
|
} else if (is_write_abort(esr)) {
|
2021-03-13 01:38:10 +08:00
|
|
|
/* It was write fault */
|
2013-09-13 06:13:39 +08:00
|
|
|
vm_flags = VM_WRITE;
|
|
|
|
|
mm_flags |= FAULT_FLAG_WRITE;
|
2021-03-13 01:38:10 +08:00
|
|
|
} else {
|
|
|
|
|
/* It was read fault */
|
|
|
|
|
vm_flags = VM_READ;
|
|
|
|
|
/* Write implies read */
|
|
|
|
|
vm_flags |= VM_WRITE;
|
|
|
|
|
/* If EPAN is absent then exec implies read */
|
|
|
|
|
if (!cpus_have_const_cap(ARM64_HAS_EPAN))
|
|
|
|
|
vm_flags |= VM_EXEC;
|
2013-09-13 06:13:39 +08:00
|
|
|
}
|
|
|
|
|
|
2018-12-28 16:30:27 +08:00
|
|
|
if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
|
2016-08-10 09:25:26 +08:00
|
|
|
if (is_el1_instruction_abort(esr))
|
2018-05-21 21:14:51 +08:00
|
|
|
die_kernel_fault("execution of user memory",
|
|
|
|
|
addr, esr, regs);
|
2016-08-10 09:25:26 +08:00
|
|
|
|
2016-02-05 22:58:48 +08:00
|
|
|
if (!search_exception_tables(regs->pc))
|
2018-05-21 21:14:51 +08:00
|
|
|
die_kernel_fault("access to user memory outside uaccess routines",
|
|
|
|
|
addr, esr, regs);
|
2016-02-05 22:58:48 +08:00
|
|
|
}
|
2015-07-23 02:05:54 +08:00
|
|
|
|
2017-06-09 01:25:28 +08:00
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
|
|
|
|
|
|
2023-02-28 01:36:29 +08:00
|
|
|
if (!(mm_flags & FAULT_FLAG_USER))
|
|
|
|
|
goto lock_mmap;
|
|
|
|
|
|
|
|
|
|
vma = lock_vma_under_rcu(mm, addr);
|
|
|
|
|
if (!vma)
|
|
|
|
|
goto lock_mmap;
|
|
|
|
|
|
|
|
|
|
if (!(vma->vm_flags & vm_flags)) {
|
|
|
|
|
vma_end_read(vma);
|
|
|
|
|
goto lock_mmap;
|
|
|
|
|
}
|
2023-05-24 21:13:05 +08:00
|
|
|
fault = handle_mm_fault(vma, addr, mm_flags | FAULT_FLAG_VMA_LOCK, regs);
|
2023-07-01 05:19:54 +08:00
|
|
|
if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
|
|
|
|
|
vma_end_read(vma);
|
2023-02-28 01:36:29 +08:00
|
|
|
|
|
|
|
|
if (!(fault & VM_FAULT_RETRY)) {
|
|
|
|
|
count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
|
|
|
|
|
goto done;
|
|
|
|
|
}
|
|
|
|
|
count_vm_vma_lock_event(VMA_LOCK_RETRY);
|
|
|
|
|
|
|
|
|
|
/* Quick path to respond to signals */
|
|
|
|
|
if (fault_signal_pending(fault, regs)) {
|
|
|
|
|
if (!user_mode(regs))
|
|
|
|
|
goto no_context;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
lock_mmap:
|
2023-06-16 08:11:44 +08:00
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
retry:
|
2023-06-16 08:11:44 +08:00
|
|
|
vma = lock_mm_and_find_vma(mm, addr, regs);
|
|
|
|
|
if (unlikely(!vma)) {
|
|
|
|
|
fault = VM_FAULT_BADMAP;
|
|
|
|
|
goto done;
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
2023-06-16 08:11:44 +08:00
|
|
|
fault = __do_page_fault(mm, vma, addr, mm_flags, vm_flags, regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2020-04-02 12:08:18 +08:00
|
|
|
/* Quick path to respond to signals */
|
|
|
|
|
if (fault_signal_pending(fault, regs)) {
|
|
|
|
|
if (!user_mode(regs))
|
|
|
|
|
goto no_context;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2017-06-09 01:25:28 +08:00
|
|
|
|
mm: avoid unnecessary page fault retires on shared memory types
I observed that for each of the shared file-backed page faults, we're very
likely to retry one more time for the 1st write fault upon no page. It's
because we'll need to release the mmap lock for dirty rate limit purpose
with balance_dirty_pages_ratelimited() (in fault_dirty_shared_page()).
Then after that throttling we return VM_FAULT_RETRY.
We did that probably because VM_FAULT_RETRY is the only way we can return
to the fault handler at that time telling it we've released the mmap lock.
However that's not ideal because it's very likely the fault does not need
to be retried at all since the pgtable was well installed before the
throttling, so the next continuous fault (including taking mmap read lock,
walk the pgtable, etc.) could be in most cases unnecessary.
It's not only slowing down page faults for shared file-backed, but also add
more mmap lock contention which is in most cases not needed at all.
To observe this, one could try to write to some shmem page and look at
"pgfault" value in /proc/vmstat, then we should expect 2 counts for each
shmem write simply because we retried, and vm event "pgfault" will capture
that.
To make it more efficient, add a new VM_FAULT_COMPLETED return code just to
show that we've completed the whole fault and released the lock. It's also
a hint that we should very possibly not need another fault immediately on
this page because we've just completed it.
This patch provides a ~12% perf boost on my aarch64 test VM with a simple
program sequentially dirtying 400MB shmem file being mmap()ed and these are
the time it needs:
Before: 650.980 ms (+-1.94%)
After: 569.396 ms (+-1.38%)
I believe it could help more than that.
We need some special care on GUP and the s390 pgfault handler (for gmap
code before returning from pgfault), the rest changes in the page fault
handlers should be relatively straightforward.
Another thing to mention is that mm_account_fault() does take this new
fault as a generic fault to be accounted, unlike VM_FAULT_RETRY.
I explicitly didn't touch hmm_vma_fault() and break_ksm() because they do
not handle VM_FAULT_RETRY even with existing code, so I'm literally keeping
them as-is.
Link: https://lkml.kernel.org/r/20220530183450.42886-1-peterx@redhat.com
Signed-off-by: Peter Xu <peterx@redhat.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vineet Gupta <vgupta@kernel.org>
Acked-by: Guo Ren <guoren@kernel.org>
Acked-by: Max Filippov <jcmvbkbc@gmail.com>
Acked-by: Christian Borntraeger <borntraeger@linux.ibm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Alistair Popple <apopple@nvidia.com>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk> [arm part]
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Stafford Horne <shorne@gmail.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Johannes Berg <johannes@sipsolutions.net>
Cc: Brian Cain <bcain@quicinc.com>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Richard Weinberger <richard@nod.at>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Janosch Frank <frankja@linux.ibm.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: James Bottomley <James.Bottomley@HansenPartnership.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Will Deacon <will@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Chris Zankel <chris@zankel.net>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Rich Felker <dalias@libc.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Helge Deller <deller@gmx.de>
Cc: Yoshinori Sato <ysato@users.osdn.me>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2022-05-31 02:34:50 +08:00
|
|
|
/* The fault is fully completed (including releasing mmap lock) */
|
|
|
|
|
if (fault & VM_FAULT_COMPLETED)
|
|
|
|
|
return 0;
|
|
|
|
|
|
2020-04-02 12:08:18 +08:00
|
|
|
if (fault & VM_FAULT_RETRY) {
|
2022-01-15 06:05:51 +08:00
|
|
|
mm_flags |= FAULT_FLAG_TRIED;
|
|
|
|
|
goto retry;
|
2017-06-09 01:25:28 +08:00
|
|
|
}
|
2020-06-09 12:33:25 +08:00
|
|
|
mmap_read_unlock(mm);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2023-02-28 01:36:29 +08:00
|
|
|
done:
|
2012-03-05 19:49:27 +08:00
|
|
|
/*
|
2017-06-09 01:25:28 +08:00
|
|
|
* Handle the "normal" (no error) case first.
|
2012-03-05 19:49:27 +08:00
|
|
|
*/
|
2017-06-09 01:25:28 +08:00
|
|
|
if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
|
2020-08-12 09:37:57 +08:00
|
|
|
VM_FAULT_BADACCESS))))
|
2012-03-05 19:49:27 +08:00
|
|
|
return 0;
|
|
|
|
|
|
2013-09-13 06:13:38 +08:00
|
|
|
/*
|
|
|
|
|
* If we are in kernel mode at this point, we have no context to
|
|
|
|
|
* handle this fault with.
|
|
|
|
|
*/
|
|
|
|
|
if (!user_mode(regs))
|
|
|
|
|
goto no_context;
|
|
|
|
|
|
2012-03-05 19:49:27 +08:00
|
|
|
if (fault & VM_FAULT_OOM) {
|
|
|
|
|
/*
|
|
|
|
|
* We ran out of memory, call the OOM killer, and return to
|
|
|
|
|
* userspace (which will retry the fault, or kill us if we got
|
|
|
|
|
* oom-killed).
|
|
|
|
|
*/
|
|
|
|
|
pagefault_out_of_memory();
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-22 16:16:42 +08:00
|
|
|
inf = esr_to_fault_info(esr);
|
2018-09-22 16:18:42 +08:00
|
|
|
set_thread_esr(addr, esr);
|
2012-03-05 19:49:27 +08:00
|
|
|
if (fault & VM_FAULT_SIGBUS) {
|
|
|
|
|
/*
|
|
|
|
|
* We had some memory, but were unable to successfully fix up
|
|
|
|
|
* this page fault.
|
|
|
|
|
*/
|
2020-11-21 04:33:46 +08:00
|
|
|
arm64_force_sig_fault(SIGBUS, BUS_ADRERR, far, inf->name);
|
2018-09-22 15:46:39 +08:00
|
|
|
} else if (fault & (VM_FAULT_HWPOISON_LARGE | VM_FAULT_HWPOISON)) {
|
|
|
|
|
unsigned int lsb;
|
|
|
|
|
|
|
|
|
|
lsb = PAGE_SHIFT;
|
|
|
|
|
if (fault & VM_FAULT_HWPOISON_LARGE)
|
|
|
|
|
lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
|
2018-02-20 22:53:22 +08:00
|
|
|
|
2020-11-21 04:33:46 +08:00
|
|
|
arm64_force_sig_mceerr(BUS_MCEERR_AR, far, lsb, inf->name);
|
2012-03-05 19:49:27 +08:00
|
|
|
} else {
|
|
|
|
|
/*
|
|
|
|
|
* Something tried to access memory that isn't in our memory
|
|
|
|
|
* map.
|
|
|
|
|
*/
|
2018-09-22 16:26:57 +08:00
|
|
|
arm64_force_sig_fault(SIGSEGV,
|
|
|
|
|
fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR,
|
2020-11-21 04:33:46 +08:00
|
|
|
far, inf->name);
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
no_context:
|
2017-06-09 23:35:52 +08:00
|
|
|
__do_kernel_fault(addr, esr, regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2020-11-21 04:33:46 +08:00
|
|
|
static int __kprobes do_translation_fault(unsigned long far,
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
unsigned long esr,
|
2012-03-05 19:49:27 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
2020-11-21 04:33:46 +08:00
|
|
|
unsigned long addr = untagged_addr(far);
|
|
|
|
|
|
2018-12-28 16:30:27 +08:00
|
|
|
if (is_ttbr0_addr(addr))
|
2020-11-21 04:33:46 +08:00
|
|
|
return do_page_fault(far, esr, regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2020-11-21 04:33:46 +08:00
|
|
|
do_bad_area(far, esr, regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-10 16:07:19 +08:00
|
|
|
static int copy_from_user_io(void *to, const void __user *from, unsigned long n)
|
|
|
|
|
{
|
|
|
|
|
const u8 __user *src = from;
|
|
|
|
|
u8 *dest = to;
|
|
|
|
|
|
|
|
|
|
for (; n; n--)
|
|
|
|
|
if (get_user(*dest++, src++))
|
|
|
|
|
break;
|
|
|
|
|
return n;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int copy_to_user_io(void __user *to, const void *from, unsigned long n)
|
|
|
|
|
{
|
|
|
|
|
const u8 *src = from;
|
|
|
|
|
u8 __user *dest = to;
|
|
|
|
|
|
|
|
|
|
for (; n; n--)
|
|
|
|
|
if (put_user(*src++, dest++))
|
|
|
|
|
break;
|
|
|
|
|
return n;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_load(unsigned long addr, int sz, u64 *out)
|
|
|
|
|
{
|
|
|
|
|
union {
|
|
|
|
|
u8 d8;
|
|
|
|
|
u16 d16;
|
|
|
|
|
u32 d32;
|
|
|
|
|
u64 d64;
|
|
|
|
|
char c[8];
|
|
|
|
|
} data;
|
|
|
|
|
|
|
|
|
|
if (sz != 1 && sz != 2 && sz != 4 && sz != 8)
|
|
|
|
|
return 1;
|
|
|
|
|
if (is_ttbr0_addr(addr)) {
|
|
|
|
|
if (copy_from_user_io(data.c, (const void __user *)addr, sz))
|
|
|
|
|
return 1;
|
|
|
|
|
} else
|
|
|
|
|
memcpy_fromio(data.c, (const void __iomem *)addr, sz);
|
|
|
|
|
switch (sz) {
|
|
|
|
|
case 1:
|
|
|
|
|
*out = data.d8;
|
|
|
|
|
break;
|
|
|
|
|
case 2:
|
|
|
|
|
*out = data.d16;
|
|
|
|
|
break;
|
|
|
|
|
case 4:
|
|
|
|
|
*out = data.d32;
|
|
|
|
|
break;
|
|
|
|
|
case 8:
|
|
|
|
|
*out = data.d64;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_store(unsigned long addr, int sz, u64 val)
|
|
|
|
|
{
|
|
|
|
|
union {
|
|
|
|
|
u8 d8;
|
|
|
|
|
u16 d16;
|
|
|
|
|
u32 d32;
|
|
|
|
|
u64 d64;
|
|
|
|
|
char c[8];
|
|
|
|
|
} data;
|
|
|
|
|
|
|
|
|
|
switch (sz) {
|
|
|
|
|
case 1:
|
|
|
|
|
data.d8 = val;
|
|
|
|
|
break;
|
|
|
|
|
case 2:
|
|
|
|
|
data.d16 = val;
|
|
|
|
|
break;
|
|
|
|
|
case 4:
|
|
|
|
|
data.d32 = val;
|
|
|
|
|
break;
|
|
|
|
|
case 8:
|
|
|
|
|
data.d64 = val;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
if (is_ttbr0_addr(addr)) {
|
|
|
|
|
if (copy_to_user_io((void __user *)addr, data.c, sz))
|
|
|
|
|
return 1;
|
|
|
|
|
} else
|
|
|
|
|
memcpy_toio((void __iomem *)addr, data.c, sz);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_dc_zva(unsigned long addr, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
int bs = read_cpuid(DCZID_EL0) & 0xf;
|
|
|
|
|
int sz = 1 << (bs + 2);
|
|
|
|
|
|
|
|
|
|
addr &= ~(sz - 1);
|
|
|
|
|
if (is_ttbr0_addr(addr)) {
|
|
|
|
|
for (; sz; sz--) {
|
|
|
|
|
if (align_store(addr, 1, 0))
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
} else
|
2022-03-03 15:43:08 +08:00
|
|
|
memset_io((void __iomem *)addr, 0, sz);
|
2020-12-10 16:07:19 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static u64 get_vn_dt(int n, int t)
|
|
|
|
|
{
|
|
|
|
|
u64 res;
|
|
|
|
|
|
|
|
|
|
switch (n) {
|
|
|
|
|
#define V(n) \
|
|
|
|
|
case n: \
|
|
|
|
|
asm("cbnz %w1, 1f\n\t" \
|
|
|
|
|
"mov %0, v"#n".d[0]\n\t" \
|
|
|
|
|
"b 2f\n\t" \
|
|
|
|
|
"1: mov %0, v"#n".d[1]\n\t" \
|
|
|
|
|
"2:" : "=r" (res) : "r" (t)); \
|
|
|
|
|
break
|
|
|
|
|
V(0); V(1); V(2); V(3); V(4); V(5); V(6); V(7);
|
|
|
|
|
V(8); V(9); V(10); V(11); V(12); V(13); V(14); V(15);
|
|
|
|
|
V(16); V(17); V(18); V(19); V(20); V(21); V(22); V(23);
|
|
|
|
|
V(24); V(25); V(26); V(27); V(28); V(29); V(30); V(31);
|
|
|
|
|
#undef V
|
|
|
|
|
default:
|
|
|
|
|
res = 0;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void set_vn_dt(int n, int t, u64 val)
|
|
|
|
|
{
|
|
|
|
|
switch (n) {
|
|
|
|
|
#define V(n) \
|
|
|
|
|
case n: \
|
|
|
|
|
asm("cbnz %w1, 1f\n\t" \
|
|
|
|
|
"mov v"#n".d[0], %0\n\t" \
|
|
|
|
|
"b 2f\n\t" \
|
|
|
|
|
"1: mov v"#n".d[1], %0\n\t" \
|
|
|
|
|
"2:" :: "r" (val), "r" (t)); \
|
|
|
|
|
break
|
|
|
|
|
V(0); V(1); V(2); V(3); V(4); V(5); V(6); V(7);
|
|
|
|
|
V(8); V(9); V(10); V(11); V(12); V(13); V(14); V(15);
|
|
|
|
|
V(16); V(17); V(18); V(19); V(20); V(21); V(22); V(23);
|
|
|
|
|
V(24); V(25); V(26); V(27); V(28); V(29); V(30); V(31);
|
|
|
|
|
#undef V
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-03 15:43:08 +08:00
|
|
|
static u64 replicate64(u64 val, int bits)
|
|
|
|
|
{
|
|
|
|
|
switch (bits) {
|
|
|
|
|
case 8:
|
|
|
|
|
val = (val << 8) | (val & 0xff);
|
|
|
|
|
fallthrough;
|
|
|
|
|
case 16:
|
|
|
|
|
val = (val << 16) | (val & 0xffff);
|
|
|
|
|
fallthrough;
|
|
|
|
|
case 32:
|
|
|
|
|
val = (val << 32) | (val & 0xffffffff);
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return val;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static u64 elem_get(u64 hi, u64 lo, int index, int esize)
|
|
|
|
|
{
|
|
|
|
|
int shift = index * esize;
|
|
|
|
|
u64 mask = GENMASK(esize - 1, 0);
|
|
|
|
|
if (shift < 64)
|
|
|
|
|
return (lo >> shift) & mask;
|
|
|
|
|
else
|
|
|
|
|
return (hi >> (shift - 64)) & mask;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void elem_set(u64 *hi, u64 *lo, int index, int esize, u64 val)
|
|
|
|
|
{
|
|
|
|
|
int shift = index * esize;
|
|
|
|
|
u64 mask = GENMASK(esize - 1, 0);
|
|
|
|
|
if (shift < 64)
|
|
|
|
|
*lo = (*lo & ~(mask << shift)) | ((val & mask) << shift);
|
|
|
|
|
else
|
|
|
|
|
*hi = (*hi & ~(mask << (shift - 64))) | ((val & mask) << (shift - 64));
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-10 16:07:19 +08:00
|
|
|
static int align_ldst_pair(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 OPC = GENMASK(31, 30);
|
|
|
|
|
const u32 L_MASK = BIT(22);
|
|
|
|
|
|
|
|
|
|
int opc = FIELD_GET(OPC, insn);
|
|
|
|
|
int L = FIELD_GET(L_MASK, insn);
|
|
|
|
|
|
|
|
|
|
bool wback = !!(insn & BIT(23));
|
|
|
|
|
bool postindex = !(insn & BIT(24));
|
|
|
|
|
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
int t2 = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT2, insn);
|
|
|
|
|
bool is_store = !L;
|
|
|
|
|
bool is_signed = !!(opc & 1);
|
|
|
|
|
int scale = 2 + (opc >> 1);
|
|
|
|
|
int datasize = 8 << scale;
|
|
|
|
|
u64 uoffset = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_7, insn);
|
|
|
|
|
s64 offset = sign_extend64(uoffset, 6) << scale;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 data1, data2;
|
|
|
|
|
u64 dbytes;
|
|
|
|
|
|
|
|
|
|
if ((is_store && (opc & 1)) || opc == 3)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
if (wback && (t == n || t2 == n) && n != 31)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
if (!is_store && t == t2)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
dbytes = datasize / 8;
|
|
|
|
|
|
|
|
|
|
address = regs_get_register(regs, n << 3);
|
|
|
|
|
|
|
|
|
|
if (!postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
|
|
|
|
|
if (is_store) {
|
|
|
|
|
data1 = pt_regs_read_reg(regs, t);
|
|
|
|
|
data2 = pt_regs_read_reg(regs, t2);
|
|
|
|
|
if (align_store(address, dbytes, data1) ||
|
|
|
|
|
align_store(address + dbytes, dbytes, data2))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_load(address, dbytes, &data1) ||
|
|
|
|
|
align_load(address + dbytes, dbytes, &data2))
|
|
|
|
|
return 1;
|
|
|
|
|
if (is_signed) {
|
|
|
|
|
data1 = sign_extend64(data1, datasize - 1);
|
|
|
|
|
data2 = sign_extend64(data2, datasize - 1);
|
|
|
|
|
}
|
|
|
|
|
pt_regs_write_reg(regs, t, data1);
|
|
|
|
|
pt_regs_write_reg(regs, t2, data2);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (wback) {
|
|
|
|
|
if (postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
if (n == 31)
|
|
|
|
|
regs->sp = address;
|
|
|
|
|
else
|
|
|
|
|
pt_regs_write_reg(regs, n, address);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-03 15:43:08 +08:00
|
|
|
static int align_ldst_vector_single(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 Q_MASK = BIT(30);
|
|
|
|
|
const u32 L_MASK = BIT(22);
|
|
|
|
|
const u32 R_MASK = BIT(21);
|
|
|
|
|
const u32 OPCODE = GENMASK(15, 13);
|
|
|
|
|
const u32 S_MASK = BIT(12);
|
|
|
|
|
const u32 SIZE = GENMASK(11, 10);
|
|
|
|
|
u32 Q = FIELD_GET(Q_MASK, insn);
|
|
|
|
|
u32 L = FIELD_GET(L_MASK, insn);
|
|
|
|
|
u32 R = FIELD_GET(R_MASK, insn);
|
|
|
|
|
u32 opcode = FIELD_GET(OPCODE, insn);
|
|
|
|
|
u32 S = FIELD_GET(S_MASK, insn);
|
|
|
|
|
u32 size = FIELD_GET(SIZE, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int m = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RM, insn);
|
|
|
|
|
bool wback = !!(insn & BIT(23));
|
|
|
|
|
int init_scale = opcode >> 1;
|
|
|
|
|
int scale = init_scale;
|
|
|
|
|
int selem = (((opcode & 1) << 1) | R) + 1;
|
|
|
|
|
bool replicate = false;
|
|
|
|
|
int index;
|
|
|
|
|
int datasize;
|
|
|
|
|
int esize;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 offs;
|
|
|
|
|
u64 rval_d0, rval_d1;
|
|
|
|
|
u64 element;
|
|
|
|
|
int ebytes;
|
|
|
|
|
int s;
|
|
|
|
|
u64 data;
|
|
|
|
|
switch (scale) {
|
|
|
|
|
case 3:
|
|
|
|
|
if (!L || S)
|
|
|
|
|
return 1;
|
|
|
|
|
scale = size;
|
|
|
|
|
replicate = true;
|
|
|
|
|
break;
|
|
|
|
|
case 0:
|
|
|
|
|
index = (Q << 3) | (S << 2) | size;
|
|
|
|
|
break;
|
|
|
|
|
case 1:
|
|
|
|
|
if (size & 1)
|
|
|
|
|
return 1;
|
|
|
|
|
index = (Q << 2) | (S << 1) | (size >> 1);
|
|
|
|
|
break;
|
|
|
|
|
case 2:
|
|
|
|
|
if (size & 2)
|
|
|
|
|
return 1;
|
|
|
|
|
if (!(size & 1))
|
|
|
|
|
index = (Q << 1) | S;
|
|
|
|
|
else {
|
|
|
|
|
if (S)
|
|
|
|
|
return 1;
|
|
|
|
|
index = Q;
|
|
|
|
|
scale = 3;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
datasize = Q ? 128 : 64;
|
|
|
|
|
esize = 8 << scale;
|
|
|
|
|
ebytes = esize / 8;
|
|
|
|
|
address = regs_get_register(regs, n << 3);
|
|
|
|
|
offs = 0;
|
|
|
|
|
if (replicate) {
|
|
|
|
|
for (s = 0; s < selem; s++) {
|
|
|
|
|
if (align_load(address + offs, ebytes, &element))
|
|
|
|
|
return 1;
|
|
|
|
|
data = replicate64(element, esize);
|
|
|
|
|
set_vn_dt(t, 0, data);
|
|
|
|
|
if (datasize == 128)
|
|
|
|
|
set_vn_dt(t, 1, data);
|
|
|
|
|
else
|
|
|
|
|
set_vn_dt(t, 1, 0);
|
|
|
|
|
offs += ebytes;
|
|
|
|
|
t = (t + 1) & 31;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
for (s = 0; s < selem; s++) {
|
|
|
|
|
rval_d0 = get_vn_dt(t, 0);
|
|
|
|
|
rval_d1 = get_vn_dt(t, 1);
|
|
|
|
|
if (L) {
|
|
|
|
|
if (align_load(address + offs, ebytes, &data))
|
|
|
|
|
return 1;
|
|
|
|
|
elem_set(&rval_d1, &rval_d0, index, esize, data);
|
|
|
|
|
set_vn_dt(t, 0, rval_d0);
|
|
|
|
|
set_vn_dt(t, 1, rval_d1);
|
|
|
|
|
} else {
|
|
|
|
|
data = elem_get(rval_d1, rval_d0, index, esize);
|
|
|
|
|
if (align_store(address + offs, ebytes, data))
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
offs += ebytes;
|
|
|
|
|
t = (t + 1) & 31;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (wback) {
|
|
|
|
|
if (m != 31)
|
|
|
|
|
offs = regs_get_register(regs, m << 3);
|
|
|
|
|
if (n == 31)
|
|
|
|
|
regs->sp = address + offs;
|
|
|
|
|
else
|
|
|
|
|
pt_regs_write_reg(regs, n, address + offs);
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-10 16:07:19 +08:00
|
|
|
static int align_ldst_pair_simdfp(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 OPC = GENMASK(31, 30);
|
|
|
|
|
const u32 L_MASK = BIT(22);
|
|
|
|
|
|
|
|
|
|
int opc = FIELD_GET(OPC, insn);
|
|
|
|
|
int L = FIELD_GET(L_MASK, insn);
|
|
|
|
|
|
|
|
|
|
bool wback = !!(insn & BIT(23));
|
|
|
|
|
bool postindex = !(insn & BIT(24));
|
|
|
|
|
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
int t2 = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT2, insn);
|
|
|
|
|
bool is_store = !L;
|
|
|
|
|
int scale = 2 + opc;
|
|
|
|
|
int datasize = 8 << scale;
|
|
|
|
|
u64 uoffset = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_7, insn);
|
|
|
|
|
s64 offset = sign_extend64(uoffset, 6) << scale;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 data1_d0, data1_d1, data2_d0, data2_d1;
|
|
|
|
|
u64 dbytes;
|
|
|
|
|
|
|
|
|
|
if (opc == 0x3)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
if (!is_store && t == t2)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
dbytes = datasize / 8;
|
|
|
|
|
|
|
|
|
|
address = regs_get_register(regs, n << 3);
|
|
|
|
|
|
|
|
|
|
if (!postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
|
|
|
|
|
if (is_store) {
|
|
|
|
|
data1_d0 = get_vn_dt(t, 0);
|
|
|
|
|
data2_d0 = get_vn_dt(t2, 0);
|
|
|
|
|
if (datasize == 128) {
|
|
|
|
|
data1_d1 = get_vn_dt(t, 1);
|
|
|
|
|
data2_d1 = get_vn_dt(t2, 1);
|
|
|
|
|
if (align_store(address, 8, data1_d0) ||
|
|
|
|
|
align_store(address + 8, 8, data1_d1) ||
|
|
|
|
|
align_store(address + 16, 8, data2_d0) ||
|
|
|
|
|
align_store(address + 24, 8, data2_d1))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_store(address, dbytes, data1_d0) ||
|
|
|
|
|
align_store(address + dbytes, dbytes, data2_d0))
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (datasize == 128) {
|
|
|
|
|
if (align_load(address, 8, &data1_d0) ||
|
|
|
|
|
align_load(address + 8, 8, &data1_d1) ||
|
|
|
|
|
align_load(address + 16, 8, &data2_d0) ||
|
|
|
|
|
align_load(address + 24, 8, &data2_d1))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_load(address, dbytes, &data1_d0) ||
|
|
|
|
|
align_load(address + dbytes, dbytes, &data2_d0))
|
|
|
|
|
return 1;
|
|
|
|
|
data1_d1 = data2_d1 = 0;
|
|
|
|
|
}
|
|
|
|
|
set_vn_dt(t, 0, data1_d0);
|
|
|
|
|
set_vn_dt(t, 1, data1_d1);
|
|
|
|
|
set_vn_dt(t2, 0, data2_d0);
|
|
|
|
|
set_vn_dt(t2, 1, data2_d1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (wback) {
|
|
|
|
|
if (postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
if (n == 31)
|
|
|
|
|
regs->sp = address;
|
|
|
|
|
else
|
|
|
|
|
pt_regs_write_reg(regs, n, address);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_ldst_regoff(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 SIZE = GENMASK(31, 30);
|
|
|
|
|
const u32 OPC = GENMASK(23, 22);
|
|
|
|
|
const u32 OPTION = GENMASK(15, 13);
|
|
|
|
|
const u32 S = BIT(12);
|
|
|
|
|
|
|
|
|
|
u32 size = FIELD_GET(SIZE, insn);
|
|
|
|
|
u32 opc = FIELD_GET(OPC, insn);
|
|
|
|
|
u32 option = FIELD_GET(OPTION, insn);
|
|
|
|
|
u32 s = FIELD_GET(S, insn);
|
|
|
|
|
int scale = size;
|
|
|
|
|
int extend_len = (option & 0x1) ? 64 : 32;
|
|
|
|
|
bool extend_unsigned = !(option & 0x4);
|
|
|
|
|
int shift = s ? scale : 0;
|
|
|
|
|
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
int m = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RM, insn);
|
|
|
|
|
bool is_store;
|
|
|
|
|
bool is_signed;
|
|
|
|
|
int regsize;
|
|
|
|
|
int datasize;
|
|
|
|
|
u64 offset;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 data;
|
|
|
|
|
|
|
|
|
|
if ((opc & 0x2) == 0) {
|
|
|
|
|
/* store or zero-extending load */
|
|
|
|
|
is_store = !(opc & 0x1);
|
|
|
|
|
regsize = size == 0x3 ? 64 : 32;
|
|
|
|
|
is_signed = false;
|
|
|
|
|
} else {
|
|
|
|
|
if (size == 0x3) {
|
|
|
|
|
if ((opc & 0x1) == 0) {
|
|
|
|
|
/* prefetch */
|
|
|
|
|
return 0;
|
|
|
|
|
} else {
|
|
|
|
|
/* undefined */
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
/* sign-extending load */
|
|
|
|
|
is_store = false;
|
|
|
|
|
if (size == 0x2 && (opc & 0x1) == 0x1) {
|
|
|
|
|
/* undefined */
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
regsize = (opc & 0x1) == 0x1 ? 32 : 64;
|
|
|
|
|
is_signed = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
datasize = 8 << scale;
|
|
|
|
|
|
|
|
|
|
if (n == t && n != 31)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
offset = pt_regs_read_reg(regs, m);
|
|
|
|
|
if (extend_len == 32) {
|
|
|
|
|
offset &= (u32)~0;
|
|
|
|
|
if (!extend_unsigned)
|
|
|
|
|
sign_extend64(offset, 31);
|
|
|
|
|
}
|
|
|
|
|
offset <<= shift;
|
|
|
|
|
|
|
|
|
|
address = regs_get_register(regs, n << 3) + offset;
|
|
|
|
|
|
|
|
|
|
if (is_store) {
|
|
|
|
|
data = pt_regs_read_reg(regs, t);
|
|
|
|
|
if (align_store(address, datasize / 8, data))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_load(address, datasize / 8, &data))
|
|
|
|
|
return 1;
|
|
|
|
|
if (is_signed) {
|
|
|
|
|
if (regsize == 32)
|
|
|
|
|
data = sign_extend32(data, datasize - 1);
|
|
|
|
|
else
|
|
|
|
|
data = sign_extend64(data, datasize - 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_ldst_regoff_simdfp(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 SIZE = GENMASK(31, 30);
|
|
|
|
|
const u32 OPC = GENMASK(23, 22);
|
|
|
|
|
const u32 OPTION = GENMASK(15, 13);
|
|
|
|
|
const u32 S = BIT(12);
|
|
|
|
|
|
|
|
|
|
u32 size = FIELD_GET(SIZE, insn);
|
|
|
|
|
u32 opc = FIELD_GET(OPC, insn);
|
|
|
|
|
u32 option = FIELD_GET(OPTION, insn);
|
|
|
|
|
u32 s = FIELD_GET(S, insn);
|
|
|
|
|
int scale = (opc & 0x2) << 1 | size;
|
|
|
|
|
int extend_len = (option & 0x1) ? 64 : 32;
|
|
|
|
|
bool extend_unsigned = !(option & 0x4);
|
|
|
|
|
int shift = s ? scale : 0;
|
|
|
|
|
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
int m = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RM, insn);
|
|
|
|
|
bool is_store = !(opc & BIT(0));
|
|
|
|
|
int datasize;
|
|
|
|
|
u64 offset;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 data_d0, data_d1;
|
|
|
|
|
|
|
|
|
|
if ((opc & 0x2) == 0)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
datasize = 8 << scale;
|
|
|
|
|
|
|
|
|
|
if (n == t && n != 31)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
offset = pt_regs_read_reg(regs, m);
|
|
|
|
|
if (extend_len == 32) {
|
|
|
|
|
offset &= (u32)~0;
|
|
|
|
|
if (!extend_unsigned)
|
|
|
|
|
sign_extend64(offset, 31);
|
|
|
|
|
}
|
|
|
|
|
offset <<= shift;
|
|
|
|
|
|
|
|
|
|
address = regs_get_register(regs, n << 3) + offset;
|
|
|
|
|
|
|
|
|
|
if (is_store) {
|
|
|
|
|
data_d0 = get_vn_dt(t, 0);
|
|
|
|
|
if (datasize == 128) {
|
|
|
|
|
data_d1 = get_vn_dt(t, 1);
|
|
|
|
|
if (align_store(address, 8, data_d0) ||
|
|
|
|
|
align_store(address + 8, 8, data_d1))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_store(address, datasize / 8, data_d0))
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (datasize == 128) {
|
|
|
|
|
if (align_load(address, 8, &data_d0) ||
|
|
|
|
|
align_load(address + 8, 8, &data_d1))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_load(address, datasize / 8, &data_d0))
|
|
|
|
|
return 1;
|
|
|
|
|
data_d1 = 0;
|
|
|
|
|
}
|
|
|
|
|
set_vn_dt(t, 0, data_d0);
|
|
|
|
|
set_vn_dt(t, 1, data_d1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_ldst_imm(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 SIZE = GENMASK(31, 30);
|
|
|
|
|
const u32 OPC = GENMASK(23, 22);
|
|
|
|
|
|
|
|
|
|
u32 size = FIELD_GET(SIZE, insn);
|
|
|
|
|
u32 opc = FIELD_GET(OPC, insn);
|
|
|
|
|
bool wback = !(insn & BIT(24)) && !!(insn & BIT(10));
|
|
|
|
|
bool postindex = wback && !(insn & BIT(11));
|
|
|
|
|
int scale = size;
|
|
|
|
|
u64 offset;
|
|
|
|
|
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
bool is_store;
|
|
|
|
|
bool is_signed;
|
|
|
|
|
int regsize;
|
|
|
|
|
int datasize;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 data;
|
|
|
|
|
|
|
|
|
|
if (!(insn & BIT(24))) {
|
|
|
|
|
u64 uoffset =
|
|
|
|
|
aarch64_insn_decode_immediate(AARCH64_INSN_IMM_9, insn);
|
|
|
|
|
offset = sign_extend64(uoffset, 8);
|
|
|
|
|
} else {
|
|
|
|
|
offset = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_12, insn);
|
|
|
|
|
offset <<= scale;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ((opc & 0x2) == 0) {
|
|
|
|
|
/* store or zero-extending load */
|
|
|
|
|
is_store = !(opc & 0x1);
|
|
|
|
|
regsize = size == 0x3 ? 64 : 32;
|
|
|
|
|
is_signed = false;
|
|
|
|
|
} else {
|
|
|
|
|
if (size == 0x3) {
|
|
|
|
|
if (FIELD_GET(GENMASK(11, 10), insn) == 0 && (opc & 0x1) == 0) {
|
|
|
|
|
/* prefetch */
|
|
|
|
|
return 0;
|
|
|
|
|
} else {
|
|
|
|
|
/* undefined */
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
/* sign-extending load */
|
|
|
|
|
is_store = false;
|
|
|
|
|
if (size == 0x2 && (opc & 0x1) == 0x1) {
|
|
|
|
|
/* undefined */
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
regsize = (opc & 0x1) == 0x1 ? 32 : 64;
|
|
|
|
|
is_signed = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
datasize = 8 << scale;
|
|
|
|
|
|
|
|
|
|
if (n == t && n != 31)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
address = regs_get_register(regs, n << 3);
|
|
|
|
|
|
|
|
|
|
if (!postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
|
|
|
|
|
if (is_store) {
|
|
|
|
|
data = pt_regs_read_reg(regs, t);
|
|
|
|
|
if (align_store(address, datasize / 8, data))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_load(address, datasize / 8, &data))
|
|
|
|
|
return 1;
|
|
|
|
|
if (is_signed) {
|
|
|
|
|
if (regsize == 32)
|
|
|
|
|
data = sign_extend32(data, datasize - 1);
|
|
|
|
|
else
|
|
|
|
|
data = sign_extend64(data, datasize - 1);
|
|
|
|
|
}
|
|
|
|
|
pt_regs_write_reg(regs, t, data);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (wback) {
|
|
|
|
|
if (postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
if (n == 31)
|
|
|
|
|
regs->sp = address;
|
|
|
|
|
else
|
|
|
|
|
pt_regs_write_reg(regs, n, address);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_ldst_imm_simdfp(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 SIZE = GENMASK(31, 30);
|
|
|
|
|
const u32 OPC = GENMASK(23, 22);
|
|
|
|
|
|
|
|
|
|
u32 size = FIELD_GET(SIZE, insn);
|
|
|
|
|
u32 opc = FIELD_GET(OPC, insn);
|
|
|
|
|
bool wback = !(insn & BIT(24)) && !!(insn & BIT(10));
|
|
|
|
|
bool postindex = wback && !(insn & BIT(11));
|
|
|
|
|
int scale = (opc & 0x2) << 1 | size;
|
|
|
|
|
u64 offset;
|
|
|
|
|
|
|
|
|
|
int n = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, insn);
|
|
|
|
|
int t = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
|
|
|
|
|
bool is_store = !(opc & BIT(0)) ;
|
|
|
|
|
int datasize;
|
|
|
|
|
u64 address;
|
|
|
|
|
u64 data_d0, data_d1;
|
|
|
|
|
|
|
|
|
|
if (scale > 4)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
if (!(insn & BIT(24))) {
|
|
|
|
|
u64 uoffset =
|
|
|
|
|
aarch64_insn_decode_immediate(AARCH64_INSN_IMM_9, insn);
|
|
|
|
|
offset = sign_extend64(uoffset, 8);
|
|
|
|
|
} else {
|
|
|
|
|
offset = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_12, insn);
|
|
|
|
|
offset <<= scale;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
datasize = 8 << scale;
|
|
|
|
|
|
|
|
|
|
address = regs_get_register(regs, n << 3);
|
|
|
|
|
|
|
|
|
|
if (!postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
|
|
|
|
|
if (is_store) {
|
|
|
|
|
data_d0 = get_vn_dt(t, 0);
|
|
|
|
|
if (datasize == 128) {
|
|
|
|
|
data_d1 = get_vn_dt(t, 1);
|
|
|
|
|
if (align_store(address, 8, data_d0) ||
|
|
|
|
|
align_store(address + 8, 8, data_d1))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_store(address, datasize / 8, data_d0))
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (datasize == 128) {
|
|
|
|
|
if (align_load(address, 8, &data_d0) ||
|
|
|
|
|
align_load(address + 8, 8, &data_d1))
|
|
|
|
|
return 1;
|
|
|
|
|
} else {
|
|
|
|
|
if (align_load(address, datasize / 8, &data_d0))
|
|
|
|
|
return 1;
|
|
|
|
|
data_d1 = 0;
|
|
|
|
|
}
|
|
|
|
|
set_vn_dt(t, 0, data_d0);
|
|
|
|
|
set_vn_dt(t, 1, data_d1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (wback) {
|
|
|
|
|
if (postindex)
|
|
|
|
|
address += offset;
|
|
|
|
|
if (n == 31)
|
|
|
|
|
regs->sp = address;
|
|
|
|
|
else
|
|
|
|
|
pt_regs_write_reg(regs, n, address);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int align_ldst(u32 insn, struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
const u32 op0 = FIELD_GET(GENMASK(31, 28), insn);
|
|
|
|
|
const u32 op1 = FIELD_GET(BIT(26), insn);
|
|
|
|
|
const u32 op2 = FIELD_GET(GENMASK(24, 23), insn);
|
|
|
|
|
const u32 op3 = FIELD_GET(GENMASK(21, 16), insn);
|
|
|
|
|
const u32 op4 = FIELD_GET(GENMASK(11, 10), insn);
|
|
|
|
|
|
|
|
|
|
if ((op0 & 0x3) == 0x2) {
|
|
|
|
|
/*
|
|
|
|
|
* |------+-----+-----+-----+-----+-----------------------------------------|
|
|
|
|
|
* | op0 | op1 | op2 | op3 | op4 | Decode group |
|
|
|
|
|
* |------+-----+-----+-----+-----+-----------------------------------------|
|
|
|
|
|
* | xx10 | - | 00 | - | - | Load/store no-allocate pair (offset) |
|
|
|
|
|
* | xx10 | - | 01 | - | - | Load/store register pair (post-indexed) |
|
|
|
|
|
* | xx10 | - | 10 | - | - | Load/store register pair (offset) |
|
|
|
|
|
* | xx10 | - | 11 | - | - | Load/store register pair (pre-indexed) |
|
|
|
|
|
* |------+-----+-----+-----+-----+-----------------------------------------|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
if (op1 == 0) { /* V == 0 */
|
|
|
|
|
/* general */
|
|
|
|
|
return align_ldst_pair(insn, regs);
|
|
|
|
|
} else {
|
|
|
|
|
/* simdfp */
|
|
|
|
|
return align_ldst_pair_simdfp(insn, regs);
|
|
|
|
|
}
|
|
|
|
|
} else if ((op0 & 0x3) == 0x3 &&
|
|
|
|
|
(((op2 & 0x2) == 0 && (op3 & 0x20) == 0 && op4 != 0x2) ||
|
|
|
|
|
((op2 & 0x2) == 0x2))) {
|
|
|
|
|
/*
|
|
|
|
|
* |------+-----+-----+--------+-----+----------------------------------------------|
|
|
|
|
|
* | op0 | op1 | op2 | op3 | op4 | Decode group |
|
|
|
|
|
* |------+-----+-----+--------+-----+----------------------------------------------|
|
|
|
|
|
* | xx11 | - | 0x | 0xxxxx | 00 | Load/store register (unscaled immediate) |
|
|
|
|
|
* | xx11 | - | 0x | 0xxxxx | 01 | Load/store register (immediate post-indexed) |
|
|
|
|
|
* | xx11 | - | 0x | 0xxxxx | 11 | Load/store register (immediate pre-indexed) |
|
|
|
|
|
* | xx11 | - | 1x | - | - | Load/store register (unsigned immediate) |
|
|
|
|
|
* |------+-----+-----+--------+-----+----------------------------------------------|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
if (op1 == 0) { /* V == 0 */
|
|
|
|
|
/* general */
|
|
|
|
|
return align_ldst_imm(insn, regs);
|
|
|
|
|
} else {
|
|
|
|
|
/* simdfp */
|
|
|
|
|
return align_ldst_imm_simdfp(insn, regs);
|
|
|
|
|
}
|
|
|
|
|
} else if ((op0 & 0x3) == 0x3 && (op2 & 0x2) == 0 &&
|
|
|
|
|
(op3 & 0x20) == 0x20 && op4 == 0x2) {
|
|
|
|
|
/*
|
|
|
|
|
* |------+-----+-----+--------+-----+---------------------------------------|
|
|
|
|
|
* | op0 | op1 | op2 | op3 | op4 | |
|
|
|
|
|
* |------+-----+-----+--------+-----+---------------------------------------|
|
|
|
|
|
* | xx11 | - | 0x | 1xxxxx | 10 | Load/store register (register offset) |
|
|
|
|
|
* |------+-----+-----+--------+-----+---------------------------------------|
|
|
|
|
|
*/
|
|
|
|
|
if (op1 == 0) { /* V == 0 */
|
|
|
|
|
/* general */
|
|
|
|
|
return align_ldst_regoff(insn, regs);
|
|
|
|
|
} else {
|
|
|
|
|
/* simdfp */
|
|
|
|
|
return align_ldst_regoff_simdfp(insn, regs);
|
|
|
|
|
}
|
2022-03-03 15:43:08 +08:00
|
|
|
} else if ((op0 & 0xb) == 0 && op1 == 1 &&
|
|
|
|
|
((op2 == 2 && ((op3 & 0x1f) == 0)) || op2 == 3)) {
|
|
|
|
|
/*
|
|
|
|
|
* |------+-----+-----+--------+-----+-------------------------------------------|
|
|
|
|
|
* | op0 | op1 | op2 | op3 | op4 | |
|
|
|
|
|
* |------+-----+-----+--------+-----+-------------------------------------------|
|
|
|
|
|
* | 0x00 | 1 | 10 | x00000 | - | Advanced SIMD load/store single structure |
|
|
|
|
|
* | 0x00 | 1 | 11 | - | - | Advanced SIMD load/store single structure |
|
|
|
|
|
* | | | | | | (post-indexed) |
|
|
|
|
|
* |------+-----+-----+--------+-----+-------------------------------------------|
|
|
|
|
|
*/
|
|
|
|
|
return align_ldst_vector_single(insn, regs);
|
|
|
|
|
|
2020-12-10 16:07:19 +08:00
|
|
|
} else
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
enum aarch64_insn_encoding_class {
|
|
|
|
|
AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
|
|
|
|
|
AARCH64_INSN_CLS_SVE, /* SVE instructions */
|
|
|
|
|
AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
|
|
|
|
|
AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
|
|
|
|
|
AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
|
|
|
|
|
AARCH64_INSN_CLS_LDST, /* Loads and stores */
|
|
|
|
|
AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and
|
|
|
|
|
system instructions */
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const int aarch64_insn_encoding_class[] = {
|
|
|
|
|
AARCH64_INSN_CLS_UNKNOWN,
|
|
|
|
|
AARCH64_INSN_CLS_UNKNOWN,
|
|
|
|
|
AARCH64_INSN_CLS_SVE,
|
|
|
|
|
AARCH64_INSN_CLS_UNKNOWN,
|
|
|
|
|
AARCH64_INSN_CLS_LDST,
|
|
|
|
|
AARCH64_INSN_CLS_DP_REG,
|
|
|
|
|
AARCH64_INSN_CLS_LDST,
|
|
|
|
|
AARCH64_INSN_CLS_DP_FPSIMD,
|
|
|
|
|
AARCH64_INSN_CLS_DP_IMM,
|
|
|
|
|
AARCH64_INSN_CLS_DP_IMM,
|
|
|
|
|
AARCH64_INSN_CLS_BR_SYS,
|
|
|
|
|
AARCH64_INSN_CLS_BR_SYS,
|
|
|
|
|
AARCH64_INSN_CLS_LDST,
|
|
|
|
|
AARCH64_INSN_CLS_DP_REG,
|
|
|
|
|
AARCH64_INSN_CLS_LDST,
|
|
|
|
|
AARCH64_INSN_CLS_DP_FPSIMD,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
|
|
|
|
|
{
|
|
|
|
|
return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-12 20:09:54 +08:00
|
|
|
#include "pcie_unalign_access.c"
|
|
|
|
|
|
2020-12-10 16:07:19 +08:00
|
|
|
static int fixup_alignment(unsigned long addr, unsigned int esr,
|
|
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
u32 insn;
|
|
|
|
|
int res;
|
2022-04-12 20:09:54 +08:00
|
|
|
struct pt_regs t = *regs;
|
|
|
|
|
int type;
|
2020-12-10 16:07:19 +08:00
|
|
|
|
|
|
|
|
if (user_mode(regs)) {
|
|
|
|
|
__le32 insn_le;
|
|
|
|
|
|
|
|
|
|
if (!is_ttbr0_addr(addr))
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
if (get_user(insn_le,
|
|
|
|
|
(__le32 __user *)instruction_pointer(regs)))
|
|
|
|
|
return 1;
|
|
|
|
|
insn = le32_to_cpu(insn_le);
|
|
|
|
|
} else {
|
|
|
|
|
if (aarch64_insn_read((void *)instruction_pointer(regs), &insn))
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-12 20:09:54 +08:00
|
|
|
pr_debug("start to handle insn:%x\n", insn);
|
|
|
|
|
|
|
|
|
|
type = aarch64_get_insn_class(insn);
|
|
|
|
|
switch (type) {
|
2020-12-10 16:07:19 +08:00
|
|
|
case AARCH64_INSN_CLS_BR_SYS:
|
|
|
|
|
if (aarch64_insn_is_dc_zva(insn))
|
|
|
|
|
res = align_dc_zva(addr, regs);
|
|
|
|
|
else
|
|
|
|
|
res = 1;
|
|
|
|
|
break;
|
|
|
|
|
case AARCH64_INSN_CLS_LDST:
|
|
|
|
|
res = align_ldst(insn, regs);
|
2022-04-12 20:09:54 +08:00
|
|
|
if (res) {
|
|
|
|
|
/* Try it again, copy back if we succeed */
|
|
|
|
|
res = align_ldst_new(insn, &t);
|
|
|
|
|
if (!res)
|
|
|
|
|
*regs = t;
|
|
|
|
|
}
|
2020-12-10 16:07:19 +08:00
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
res = 1;
|
|
|
|
|
}
|
|
|
|
|
if (!res) {
|
|
|
|
|
instruction_pointer_set(regs, instruction_pointer(regs) + 4);
|
2022-04-12 20:09:54 +08:00
|
|
|
} else {
|
|
|
|
|
pr_err("cannot handle insn:%x, type:%d\n", insn, type);
|
|
|
|
|
dump_stack();
|
2020-12-10 16:07:19 +08:00
|
|
|
}
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static int do_alignment_fault(unsigned long far, unsigned long esr,
|
2016-02-16 12:44:35 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
2022-07-01 21:53:22 +08:00
|
|
|
if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS) &&
|
|
|
|
|
compat_user_mode(regs))
|
|
|
|
|
return do_compat_alignment_fixup(far, regs);
|
2020-12-10 16:07:19 +08:00
|
|
|
#ifdef CONFIG_ALTRA_ERRATUM_82288
|
|
|
|
|
if (!fixup_alignment(far, esr, regs))
|
|
|
|
|
return 0;
|
|
|
|
|
#endif
|
2020-11-21 04:33:46 +08:00
|
|
|
do_bad_area(far, esr, regs);
|
2016-02-16 12:44:35 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2017-09-22 18:01:26 +08:00
|
|
|
return 1; /* "fault" */
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs)
|
2017-06-22 02:17:08 +08:00
|
|
|
{
|
|
|
|
|
const struct fault_info *inf;
|
2020-11-21 04:33:46 +08:00
|
|
|
unsigned long siaddr;
|
2017-06-22 02:17:08 +08:00
|
|
|
|
|
|
|
|
inf = esr_to_fault_info(esr);
|
|
|
|
|
|
2020-05-02 00:45:43 +08:00
|
|
|
if (user_mode(regs) && apei_claim_sea(regs) == 0) {
|
|
|
|
|
/*
|
|
|
|
|
* APEI claimed this as a firmware-first notification.
|
|
|
|
|
* Some processing deferred to task_work before ret_to_user().
|
|
|
|
|
*/
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2017-06-22 02:17:09 +08:00
|
|
|
|
2020-11-21 04:33:46 +08:00
|
|
|
if (esr & ESR_ELx_FnV) {
|
|
|
|
|
siaddr = 0;
|
|
|
|
|
} else {
|
|
|
|
|
/*
|
|
|
|
|
* The architecture specifies that the tag bits of FAR_EL1 are
|
|
|
|
|
* UNKNOWN for synchronous external aborts. Mask them out now
|
|
|
|
|
* so that userspace doesn't see them.
|
|
|
|
|
*/
|
|
|
|
|
siaddr = untagged_addr(far);
|
|
|
|
|
}
|
2018-09-21 23:24:40 +08:00
|
|
|
arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
|
2017-06-22 02:17:08 +08:00
|
|
|
|
2017-12-13 18:36:47 +08:00
|
|
|
return 0;
|
2017-06-22 02:17:08 +08:00
|
|
|
}
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
static int do_tag_check_fault(unsigned long far, unsigned long esr,
|
2019-09-16 18:51:17 +08:00
|
|
|
struct pt_regs *regs)
|
|
|
|
|
{
|
2020-11-21 04:33:46 +08:00
|
|
|
/*
|
2021-01-16 01:41:53 +08:00
|
|
|
* The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN
|
|
|
|
|
* for tag check faults. Set them to corresponding bits in the untagged
|
|
|
|
|
* address.
|
2020-11-21 04:33:46 +08:00
|
|
|
*/
|
2021-01-16 01:41:53 +08:00
|
|
|
far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK);
|
2020-11-21 04:33:46 +08:00
|
|
|
do_bad_area(far, esr, regs);
|
2019-09-16 18:51:17 +08:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-04 13:51:01 +08:00
|
|
|
static const struct fault_info fault_info[] = {
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "ttbr address size fault" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "level 1 address size fault" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "level 2 address size fault" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "level 3 address size fault" },
|
2014-11-21 22:22:22 +08:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
|
2012-03-05 19:49:27 +08:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
|
|
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
|
2017-09-29 19:27:41 +08:00
|
|
|
{ do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 8" },
|
2013-04-10 20:48:00 +08:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
|
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
|
2012-03-05 19:49:27 +08:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 12" },
|
2013-04-10 20:48:00 +08:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
|
|
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
|
2012-03-05 19:49:27 +08:00
|
|
|
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_sea, SIGBUS, BUS_OBJERR, "synchronous external abort" },
|
2019-09-16 18:51:17 +08:00
|
|
|
{ do_tag_check_fault, SIGSEGV, SEGV_MTESERR, "synchronous tag check fault" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 18" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 19" },
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 0 (translation table walk)" },
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 1 (translation table walk)" },
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 2 (translation table walk)" },
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 3 (translation table walk)" },
|
|
|
|
|
{ do_sea, SIGBUS, BUS_OBJERR, "synchronous parity or ECC error" }, // Reserved when RAS is implemented
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 25" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 26" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 27" },
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
|
|
|
|
|
{ do_sea, SIGKILL, SI_KERNEL, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 32" },
|
2016-02-16 12:44:35 +08:00
|
|
|
{ do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 34" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 35" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 36" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 37" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 38" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 39" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 40" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 41" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 42" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 43" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 44" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 45" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 46" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 47" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "TLB conflict abort" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "Unsupported atomic hardware update fault" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 50" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 51" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "implementation fault (lockdown abort)" },
|
|
|
|
|
{ do_bad, SIGBUS, BUS_OBJERR, "implementation fault (unsupported exclusive)" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 54" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 55" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 56" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 57" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 58" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 59" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 60" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "section domain fault" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "page domain fault" },
|
|
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
|
2012-03-05 19:49:27 +08:00
|
|
|
};
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2017-04-04 13:51:01 +08:00
|
|
|
const struct fault_info *inf = esr_to_fault_info(esr);
|
2020-11-21 04:33:46 +08:00
|
|
|
unsigned long addr = untagged_addr(far);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2020-11-21 04:33:46 +08:00
|
|
|
if (!inf->fn(far, esr, regs))
|
2012-03-05 19:49:27 +08:00
|
|
|
return;
|
|
|
|
|
|
arm64: mm: use die_kernel_fault() in do_mem_abort()
If we take an unhandled fault from EL1, either:
a) The xFSC handler calls die_kernel_fault() directly. In this case,
die_kernel_fault() calls:
pr_alert(..., msg, addr);
mem_abort_decode(esr);
show_pte(addr);
die();
bust_spinlocks(0);
do_exit(SIGKILL);
b) The xFSC handler returns to do_mem_abort(), indicating failure. In
this case, do_mem_abort() calls:
pr_alert(..., addr);
mem_abort_decode(esr);
show_pte(addr);
arm64_notify_die() {
die();
}
This inconstency is unfortunatem, and in theory in case (b) registered
notifiers can prevent us from terminating the faulting thread by
returning NOTIFY_STOP, whereupon we'll end up returning from the fault,
replaying, and almost certainly get stuck in a livelock spewing errors
into dmesg. We don't expect notifers to fix things up, since we dump
state to dmesg before invoking them, so it would be more sensible to
consistently terminate the thread in this case.
This patch has do_mem_abort() call die_kernel_fault() for unhandled
faults taken from EL1. Where we would previously have logged a messafe
of the form:
| Unhandled fault at ${ADDR}
... we will now log a message of the form:
| Unable to handle kernel ${FAULT_NAME} at virtual address ${ADDR}
... and we will consistently terminate the thread from which the fault
was taken.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Tested-by: Andrey Konovalov <andreyknvl@gmail.com>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211207183226.834557-2-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2021-12-08 02:32:25 +08:00
|
|
|
if (!user_mode(regs))
|
|
|
|
|
die_kernel_fault(inf->name, addr, esr, regs);
|
2017-10-19 18:19:55 +08:00
|
|
|
|
2020-11-21 04:33:46 +08:00
|
|
|
/*
|
|
|
|
|
* At this point we have an unrecognized fault type whose tag bits may
|
|
|
|
|
* have been defined as UNKNOWN. Therefore we only expose the untagged
|
|
|
|
|
* address to the signal handler.
|
|
|
|
|
*/
|
|
|
|
|
arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
2019-10-26 00:42:10 +08:00
|
|
|
NOKPROBE_SYMBOL(do_mem_abort);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2020-11-21 04:33:46 +08:00
|
|
|
arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
|
|
|
|
|
addr, esr);
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
2019-10-26 00:42:10 +08:00
|
|
|
NOKPROBE_SYMBOL(do_sp_pc_abort);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2015-07-24 23:37:48 +08:00
|
|
|
/*
|
|
|
|
|
* __refdata because early_brk64 is __init, but the reference to it is
|
|
|
|
|
* clobbered at arch_initcall time.
|
|
|
|
|
* See traps.c and debug-monitors.c:debug_traps_init().
|
|
|
|
|
*/
|
|
|
|
|
static struct fault_info __refdata debug_fault_info[] = {
|
2012-03-05 19:49:27 +08:00
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
|
|
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
|
|
|
|
|
{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 3" },
|
2012-03-05 19:49:27 +08:00
|
|
|
{ do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" },
|
2015-07-24 23:37:48 +08:00
|
|
|
{ early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
|
arm64: signal: Ensure si_code is valid for all fault signals
Currently, as reported by Eric, an invalid si_code value 0 is
passed in many signals delivered to userspace in response to faults
and other kernel errors. Typically 0 is passed when the fault is
insufficiently diagnosable or when there does not appear to be any
sensible alternative value to choose.
This appears to violate POSIX, and is intuitively wrong for at
least two reasons arising from the fact that 0 == SI_USER:
1) si_code is a union selector, and SI_USER (and si_code <= 0 in
general) implies the existence of a different set of fields
(siginfo._kill) from that which exists for a fault signal
(siginfo._sigfault). However, the code raising the signal
typically writes only the _sigfault fields, and the _kill
fields make no sense in this case.
Thus when userspace sees si_code == 0 (SI_USER) it may
legitimately inspect fields in the inactive union member _kill
and obtain garbage as a result.
There appears to be software in the wild relying on this,
albeit generally only for printing diagnostic messages.
2) Software that wants to be robust against spurious signals may
discard signals where si_code == SI_USER (or <= 0), or may
filter such signals based on the si_uid and si_pid fields of
siginfo._sigkill. In the case of fault signals, this means
that important (and usually fatal) error conditions may be
silently ignored.
In practice, many of the faults for which arm64 passes si_code == 0
are undiagnosable conditions such as exceptions with syndrome
values in ESR_ELx to which the architecture does not yet assign any
meaning, or conditions indicative of a bug or error in the kernel
or system and thus that are unrecoverable and should never occur in
normal operation.
The approach taken in this patch is to translate all such
undiagnosable or "impossible" synchronous fault conditions to
SIGKILL, since these are at least probably localisable to a single
process. Some of these conditions should really result in a kernel
panic, but due to the lack of diagnostic information it is
difficult to be certain: this patch does not add any calls to
panic(), but this could change later if justified.
Although si_code will not reach userspace in the case of SIGKILL,
it is still desirable to pass a nonzero value so that the common
siginfo handling code can detect incorrect use of si_code == 0
without false positives. In this case the si_code dependent
siginfo fields will not be correctly initialised, but since they
are not passed to userspace I deem this not to matter.
A few faults can reasonably occur in realistic userspace scenarios,
and _should_ raise a regular, handleable (but perhaps not
ignorable/blockable) signal: for these, this patch attempts to
choose a suitable standard si_code value for the raised signal in
each case instead of 0.
arm64 was the only arch to define a BUS_FIXME code, so after this
patch nobody defines it. This patch therefore also removes the
relevant code from siginfo_layout().
Cc: James Morse <james.morse@arm.com>
Reported-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-03-09 01:41:05 +08:00
|
|
|
{ do_bad, SIGKILL, SI_KERNEL, "unknown 7" },
|
2012-03-05 19:49:27 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
void __init hook_debug_fault_code(int nr,
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
int (*fn)(unsigned long, unsigned long, struct pt_regs *),
|
2012-03-05 19:49:27 +08:00
|
|
|
int sig, int code, const char *name)
|
|
|
|
|
{
|
|
|
|
|
BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
|
|
|
|
|
|
|
|
|
|
debug_fault_info[nr].fn = fn;
|
|
|
|
|
debug_fault_info[nr].sig = sig;
|
|
|
|
|
debug_fault_info[nr].code = code;
|
|
|
|
|
debug_fault_info[nr].name = name;
|
|
|
|
|
}
|
|
|
|
|
|
2019-08-01 22:36:14 +08:00
|
|
|
/*
|
|
|
|
|
* In debug exception context, we explicitly disable preemption despite
|
|
|
|
|
* having interrupts disabled.
|
|
|
|
|
* This serves two purposes: it makes it much less likely that we would
|
|
|
|
|
* accidentally schedule in exception context and it will force a warning
|
|
|
|
|
* if we somehow manage to schedule by accident.
|
|
|
|
|
*/
|
|
|
|
|
static void debug_exception_enter(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
preempt_disable();
|
|
|
|
|
|
|
|
|
|
/* This code is a bit fragile. Test it. */
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
|
|
|
|
|
}
|
|
|
|
|
NOKPROBE_SYMBOL(debug_exception_enter);
|
|
|
|
|
|
|
|
|
|
static void debug_exception_exit(struct pt_regs *regs)
|
|
|
|
|
{
|
|
|
|
|
preempt_enable_no_resched();
|
|
|
|
|
}
|
|
|
|
|
NOKPROBE_SYMBOL(debug_exception_exit);
|
|
|
|
|
|
arm64: Treat ESR_ELx as a 64-bit register
In the initial release of the ARM Architecture Reference Manual for
ARMv8-A, the ESR_ELx registers were defined as 32-bit registers. This
changed in 2018 with version D.a (ARM DDI 0487D.a) of the architecture,
when they became 64-bit registers, with bits [63:32] defined as RES0. In
version G.a, a new field was added to ESR_ELx, ISS2, which covers bits
[36:32]. This field is used when the Armv8.7 extension FEAT_LS64 is
implemented.
As a result of the evolution of the register width, Linux stores it as
both a 64-bit value and a 32-bit value, which hasn't affected correctness
so far as Linux only uses the lower 32 bits of the register.
Make the register type consistent and always treat it as 64-bit wide. The
register is redefined as an "unsigned long", which is an unsigned
double-word (64-bit quantity) for the LP64 machine (aapcs64 [1], Table 1,
page 14). The type was chosen because "unsigned int" is the most frequent
type for ESR_ELx and because FAR_ELx, which is used together with ESR_ELx
in exception handling, is also declared as "unsigned long". The 64-bit type
also makes adding support for architectural features that use fields above
bit 31 easier in the future.
The KVM hypervisor will receive a similar update in a subsequent patch.
[1] https://github.com/ARM-software/abi-aa/releases/download/2021Q3/aapcs64.pdf
Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20220425114444.368693-4-alexandru.elisei@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-04-25 19:44:42 +08:00
|
|
|
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
|
2019-10-26 00:42:15 +08:00
|
|
|
struct pt_regs *regs)
|
2012-03-05 19:49:27 +08:00
|
|
|
{
|
2018-09-22 23:39:54 +08:00
|
|
|
const struct fault_info *inf = esr_to_debug_fault_info(esr);
|
2019-03-01 21:28:00 +08:00
|
|
|
unsigned long pc = instruction_pointer(regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2019-08-01 22:36:14 +08:00
|
|
|
debug_exception_enter(regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2019-03-01 21:28:00 +08:00
|
|
|
if (user_mode(regs) && !is_ttbr0_addr(pc))
|
2018-02-03 01:31:39 +08:00
|
|
|
arm64_apply_bp_hardening();
|
|
|
|
|
|
2019-02-25 20:06:43 +08:00
|
|
|
if (inf->fn(addr_if_watchpoint, esr, regs)) {
|
2020-11-21 04:33:46 +08:00
|
|
|
arm64_notify_die(inf->name, regs, inf->sig, inf->code, pc, esr);
|
2016-04-13 20:40:00 +08:00
|
|
|
}
|
2012-03-05 19:49:27 +08:00
|
|
|
|
2019-08-01 22:36:14 +08:00
|
|
|
debug_exception_exit(regs);
|
2012-03-05 19:49:27 +08:00
|
|
|
}
|
arm64: Kprobes with single stepping support
Add support for basic kernel probes(kprobes) and jump probes
(jprobes) for ARM64.
Kprobes utilizes software breakpoint and single step debug
exceptions supported on ARM v8.
A software breakpoint is placed at the probe address to trap the
kernel execution into the kprobe handler.
ARM v8 supports enabling single stepping before the break exception
return (ERET), with next PC in exception return address (ELR_EL1). The
kprobe handler prepares an executable memory slot for out-of-line
execution with a copy of the original instruction being probed, and
enables single stepping. The PC is set to the out-of-line slot address
before the ERET. With this scheme, the instruction is executed with the
exact same register context except for the PC (and DAIF) registers.
Debug mask (PSTATE.D) is enabled only when single stepping a recursive
kprobe, e.g.: during kprobes reenter so that probed instruction can be
single stepped within the kprobe handler -exception- context.
The recursion depth of kprobe is always 2, i.e. upon probe re-entry,
any further re-entry is prevented by not calling handlers and the case
counted as a missed kprobe).
Single stepping from the x-o-l slot has a drawback for PC-relative accesses
like branching and symbolic literals access as the offset from the new PC
(slot address) may not be ensured to fit in the immediate value of
the opcode. Such instructions need simulation, so reject
probing them.
Instructions generating exceptions or cpu mode change are rejected
for probing.
Exclusive load/store instructions are rejected too. Additionally, the
code is checked to see if it is inside an exclusive load/store sequence
(code from Pratyush).
System instructions are mostly enabled for stepping, except MSR/MRS
accesses to "DAIF" flags in PSTATE, which are not safe for
probing.
This also changes arch/arm64/include/asm/ptrace.h to use
include/asm-generic/ptrace.h.
Thanks to Steve Capper and Pratyush Anand for several suggested
Changes.
Signed-off-by: Sandeepa Prabhu <sandeepa.s.prabhu@gmail.com>
Signed-off-by: David A. Long <dave.long@linaro.org>
Signed-off-by: Pratyush Anand <panand@redhat.com>
Acked-by: Masami Hiramatsu <mhiramat@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-09 00:35:48 +08:00
|
|
|
NOKPROBE_SYMBOL(do_debug_exception);
|
2021-06-03 07:52:29 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Used during anonymous page fault handling.
|
|
|
|
|
*/
|
2023-01-17 03:18:09 +08:00
|
|
|
struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
|
2021-06-03 07:52:29 +08:00
|
|
|
unsigned long vaddr)
|
|
|
|
|
{
|
|
|
|
|
gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If the page is mapped with PROT_MTE, initialise the tags at the
|
|
|
|
|
* point of allocation and page zeroing as this is usually faster than
|
|
|
|
|
* separate DC ZVA and STGM.
|
|
|
|
|
*/
|
|
|
|
|
if (vma->vm_flags & VM_MTE)
|
|
|
|
|
flags |= __GFP_ZEROTAGS;
|
|
|
|
|
|
2023-01-17 03:18:09 +08:00
|
|
|
return vma_alloc_folio(flags, 0, vma, vaddr, false);
|
2021-06-03 07:52:29 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void tag_clear_highpage(struct page *page)
|
|
|
|
|
{
|
2022-11-04 09:10:38 +08:00
|
|
|
/* Newly allocated page, shouldn't have been tagged yet */
|
|
|
|
|
WARN_ON_ONCE(!try_page_mte_tagging(page));
|
2021-06-03 07:52:29 +08:00
|
|
|
mte_zero_clear_page_tags(page_address(page));
|
2022-11-04 09:10:35 +08:00
|
|
|
set_page_mte_tagged(page);
|
2021-06-03 07:52:29 +08:00
|
|
|
}
|