2019-06-03 13:44:50 +08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2012-03-05 19:49:32 +08:00
|
|
|
/*
|
|
|
|
|
* Based on arch/arm/include/asm/uaccess.h
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ASM_UACCESS_H
|
|
|
|
|
#define __ASM_UACCESS_H
|
|
|
|
|
|
2016-07-01 21:58:21 +08:00
|
|
|
#include <asm/alternative.h>
|
2016-07-01 23:53:00 +08:00
|
|
|
#include <asm/kernel-pgtable.h>
|
2016-07-01 21:58:21 +08:00
|
|
|
#include <asm/sysreg.h>
|
|
|
|
|
|
2012-03-05 19:49:32 +08:00
|
|
|
/*
|
|
|
|
|
* User space memory access functions
|
|
|
|
|
*/
|
2016-10-19 21:40:54 +08:00
|
|
|
#include <linux/bitops.h>
|
2016-06-09 05:40:56 +08:00
|
|
|
#include <linux/kasan-checks.h>
|
2012-03-05 19:49:32 +08:00
|
|
|
#include <linux/string.h>
|
|
|
|
|
|
arm64: extable: consolidate definitions
In subsequent patches we'll alter the structure and usage of struct
exception_table_entry. For inline assembly, we create these using the
`_ASM_EXTABLE()` CPP macro defined in <asm/uaccess.h>, and for plain
assembly code we use the `_asm_extable()` GAS macro defined in
<asm/assembler.h>, which are largely identical save for different
escaping and stringification requirements.
This patch moves the common definitions to a new <asm/asm-extable.h>
header, so that it's easier to keep the two in-sync, and to remove the
implication that these are only used for uaccess helpers (as e.g.
load_unaligned_zeropad() is only used on kernel memory, and depends upon
`_ASM_EXTABLE()`.
At the same time, a few minor modifications are made for clarity and in
preparation for subsequent patches:
* The structure creation is factored out into an `__ASM_EXTABLE_RAW()`
macro. This will make it easier to support different fixup variants in
subsequent patches without needing to update all users of
`_ASM_EXTABLE()`, and makes it easier to see tha the CPP and GAS
variants of the macros are structurally identical.
For the CPP macro, the stringification of fields is left to the
wrapper macro, `_ASM_EXTABLE()`, as in subsequent patches it will be
necessary to stringify fields in wrapper macros to safely concatenate
strings which cannot be token-pasted together in CPP.
* The fields of the structure are created separately on their own lines.
This will make it easier to add/remove/modify individual fields
clearly.
* Additional parentheses are added around the use of macro arguments in
field definitions to avoid any potential problems with evaluation due
to operator precedence, and to make errors upon misuse clearer.
* USER() is moved into <asm/asm-uaccess.h>, as it is not required by all
assembly code, and is already refered to by comments in that file.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211019160219.5202-8-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-10-20 00:02:13 +08:00
|
|
|
#include <asm/asm-extable.h>
|
2015-07-23 02:05:54 +08:00
|
|
|
#include <asm/cpufeature.h>
|
2020-06-30 20:53:07 +08:00
|
|
|
#include <asm/mmu.h>
|
2021-03-15 21:20:15 +08:00
|
|
|
#include <asm/mte.h>
|
2012-03-05 19:49:32 +08:00
|
|
|
#include <asm/ptrace.h>
|
|
|
|
|
#include <asm/memory.h>
|
2016-12-26 03:00:03 +08:00
|
|
|
#include <asm/extable.h>
|
2012-03-05 19:49:32 +08:00
|
|
|
|
uaccess: generalize access_ok()
There are many different ways that access_ok() is defined across
architectures, but in the end, they all just compare against the
user_addr_max() value or they accept anything.
Provide one definition that works for most architectures, checking
against TASK_SIZE_MAX for user processes or skipping the check inside
of uaccess_kernel() sections.
For architectures without CONFIG_SET_FS(), this should be the fastest
check, as it comes down to a single comparison of a pointer against a
compile-time constant, while the architecture specific versions tend to
do something more complex for historic reasons or get something wrong.
Type checking for __user annotations is handled inconsistently across
architectures, but this is easily simplified as well by using an inline
function that takes a 'const void __user *' argument. A handful of
callers need an extra __user annotation for this.
Some architectures had trick to use 33-bit or 65-bit arithmetic on the
addresses to calculate the overflow, however this simpler version uses
fewer registers, which means it can produce better object code in the
end despite needing a second (statically predicted) branch.
Reviewed-by: Christoph Hellwig <hch@lst.de>
Acked-by: Mark Rutland <mark.rutland@arm.com> [arm64, asm-generic]
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Stafford Horne <shorne@gmail.com>
Acked-by: Dinh Nguyen <dinguyen@kernel.org>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
2022-02-16 00:55:04 +08:00
|
|
|
static inline int __access_ok(const void __user *ptr, unsigned long size);
|
2022-02-14 16:00:10 +08:00
|
|
|
|
2012-03-05 19:49:32 +08:00
|
|
|
/*
|
|
|
|
|
* Test whether a block of memory is a valid user space address.
|
|
|
|
|
* Returns 1 if the range is valid, 0 otherwise.
|
|
|
|
|
*
|
|
|
|
|
* This is equivalent to the following test:
|
arm64: uaccess: remove set_fs()
Now that the uaccess primitives dont take addr_limit into account, we
have no need to manipulate this via set_fs() and get_fs(). Remove
support for these, along with some infrastructure this renders
redundant.
We no longer need to flip UAO to access kernel memory under KERNEL_DS,
and head.S unconditionally clears UAO for all kernel configurations via
an ERET in init_kernel_el. Thus, we don't need to dynamically flip UAO,
nor do we need to context-switch it. However, we still need to adjust
PAN during SDEI entry.
Masking of __user pointers no longer needs to use the dynamic value of
addr_limit, and can use a constant derived from the maximum possible
userspace task size. A new TASK_SIZE_MAX constant is introduced for
this, which is also used by core code. In configurations supporting
52-bit VAs, this may include a region of unusable VA space above a
48-bit TTBR0 limit, but never includes any portion of TTBR1.
Note that TASK_SIZE_MAX is an exclusive limit, while USER_DS and
KERNEL_DS were inclusive limits, and is converted to a mask by
subtracting one.
As the SDEI entry code repurposes the otherwise unnecessary
pt_regs::orig_addr_limit field to store the TTBR1 of the interrupted
context, for now we rename that to pt_regs::sdei_ttbr1. In future we can
consider factoring that out.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: James Morse <james.morse@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201202131558.39270-10-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-12-02 21:15:55 +08:00
|
|
|
* (u65)addr + (u65)size <= (u65)TASK_SIZE_MAX
|
2012-03-05 19:49:32 +08:00
|
|
|
*/
|
2022-02-14 16:00:10 +08:00
|
|
|
static inline int access_ok(const void __user *addr, unsigned long size)
|
2018-02-05 23:34:18 +08:00
|
|
|
{
|
2019-12-05 21:57:36 +08:00
|
|
|
/*
|
|
|
|
|
* Asynchronous I/O running in a kernel thread does not have the
|
|
|
|
|
* TIF_TAGGED_ADDR flag of the process owning the mm, so always untag
|
|
|
|
|
* the user address before checking.
|
|
|
|
|
*/
|
2019-07-24 01:58:39 +08:00
|
|
|
if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) &&
|
2019-12-05 21:57:36 +08:00
|
|
|
(current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR)))
|
2019-07-24 01:58:39 +08:00
|
|
|
addr = untagged_addr(addr);
|
2019-07-24 01:58:38 +08:00
|
|
|
|
2022-02-14 16:00:10 +08:00
|
|
|
return likely(__access_ok(addr, size));
|
2018-02-05 23:34:18 +08:00
|
|
|
}
|
uaccess: generalize access_ok()
There are many different ways that access_ok() is defined across
architectures, but in the end, they all just compare against the
user_addr_max() value or they accept anything.
Provide one definition that works for most architectures, checking
against TASK_SIZE_MAX for user processes or skipping the check inside
of uaccess_kernel() sections.
For architectures without CONFIG_SET_FS(), this should be the fastest
check, as it comes down to a single comparison of a pointer against a
compile-time constant, while the architecture specific versions tend to
do something more complex for historic reasons or get something wrong.
Type checking for __user annotations is handled inconsistently across
architectures, but this is easily simplified as well by using an inline
function that takes a 'const void __user *' argument. A handful of
callers need an extra __user annotation for this.
Some architectures had trick to use 33-bit or 65-bit arithmetic on the
addresses to calculate the overflow, however this simpler version uses
fewer registers, which means it can produce better object code in the
end despite needing a second (statically predicted) branch.
Reviewed-by: Christoph Hellwig <hch@lst.de>
Acked-by: Mark Rutland <mark.rutland@arm.com> [arm64, asm-generic]
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Stafford Horne <shorne@gmail.com>
Acked-by: Dinh Nguyen <dinguyen@kernel.org>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
2022-02-16 00:55:04 +08:00
|
|
|
#define access_ok access_ok
|
|
|
|
|
|
|
|
|
|
#include <asm-generic/access_ok.h>
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2016-07-01 21:58:21 +08:00
|
|
|
/*
|
|
|
|
|
* User access enabling/disabling.
|
|
|
|
|
*/
|
2016-07-01 23:53:00 +08:00
|
|
|
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
|
|
|
|
|
static inline void __uaccess_ttbr0_disable(void)
|
|
|
|
|
{
|
2018-01-10 21:18:30 +08:00
|
|
|
unsigned long flags, ttbr;
|
2016-07-01 23:53:00 +08:00
|
|
|
|
2018-01-10 21:18:30 +08:00
|
|
|
local_irq_save(flags);
|
2017-08-10 20:58:16 +08:00
|
|
|
ttbr = read_sysreg(ttbr1_el1);
|
2018-01-10 21:18:30 +08:00
|
|
|
ttbr &= ~TTBR_ASID_MASK;
|
2020-11-03 18:22:29 +08:00
|
|
|
/* reserved_pg_dir placed before swapper_pg_dir */
|
2021-02-02 20:36:57 +08:00
|
|
|
write_sysreg(ttbr - RESERVED_SWAPPER_OFFSET, ttbr0_el1);
|
2017-08-10 20:58:16 +08:00
|
|
|
isb();
|
|
|
|
|
/* Set reserved ASID */
|
|
|
|
|
write_sysreg(ttbr, ttbr1_el1);
|
2016-07-01 23:53:00 +08:00
|
|
|
isb();
|
2018-01-10 21:18:30 +08:00
|
|
|
local_irq_restore(flags);
|
2016-07-01 23:53:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void __uaccess_ttbr0_enable(void)
|
|
|
|
|
{
|
2017-08-10 20:58:16 +08:00
|
|
|
unsigned long flags, ttbr0, ttbr1;
|
2016-07-01 23:53:00 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Disable interrupts to avoid preemption between reading the 'ttbr0'
|
|
|
|
|
* variable and the MSR. A context switch could trigger an ASID
|
|
|
|
|
* roll-over and an update of 'ttbr0'.
|
|
|
|
|
*/
|
|
|
|
|
local_irq_save(flags);
|
2018-01-10 21:18:30 +08:00
|
|
|
ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
|
2017-08-10 20:58:16 +08:00
|
|
|
|
|
|
|
|
/* Restore active ASID */
|
|
|
|
|
ttbr1 = read_sysreg(ttbr1_el1);
|
2018-01-10 21:18:30 +08:00
|
|
|
ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */
|
2017-12-02 01:33:48 +08:00
|
|
|
ttbr1 |= ttbr0 & TTBR_ASID_MASK;
|
2017-08-10 20:58:16 +08:00
|
|
|
write_sysreg(ttbr1, ttbr1_el1);
|
|
|
|
|
isb();
|
|
|
|
|
|
|
|
|
|
/* Restore user page table */
|
|
|
|
|
write_sysreg(ttbr0, ttbr0_el1);
|
2016-07-01 23:53:00 +08:00
|
|
|
isb();
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool uaccess_ttbr0_disable(void)
|
|
|
|
|
{
|
|
|
|
|
if (!system_uses_ttbr0_pan())
|
|
|
|
|
return false;
|
|
|
|
|
__uaccess_ttbr0_disable();
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool uaccess_ttbr0_enable(void)
|
|
|
|
|
{
|
|
|
|
|
if (!system_uses_ttbr0_pan())
|
|
|
|
|
return false;
|
|
|
|
|
__uaccess_ttbr0_enable();
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
static inline bool uaccess_ttbr0_disable(void)
|
|
|
|
|
{
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool uaccess_ttbr0_enable(void)
|
|
|
|
|
{
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2018-01-08 23:38:11 +08:00
|
|
|
static inline void __uaccess_disable_hw_pan(void)
|
|
|
|
|
{
|
|
|
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,
|
|
|
|
|
CONFIG_ARM64_PAN));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void __uaccess_enable_hw_pan(void)
|
|
|
|
|
{
|
|
|
|
|
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
|
|
|
|
|
CONFIG_ARM64_PAN));
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-23 04:01:35 +08:00
|
|
|
/*
|
|
|
|
|
* The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0
|
|
|
|
|
* affects EL0 and TCF affects EL1 irrespective of which TTBR is
|
|
|
|
|
* used.
|
|
|
|
|
* The kernel accesses TTBR0 usually with LDTR/STTR instructions
|
|
|
|
|
* when UAO is available, so these would act as EL0 accesses using
|
|
|
|
|
* TCF0.
|
|
|
|
|
* However futex.h code uses exclusives which would be executed as
|
|
|
|
|
* EL1, this can potentially cause a tag check fault even if the
|
|
|
|
|
* user disables TCF0.
|
|
|
|
|
*
|
|
|
|
|
* To address the problem we set the PSTATE.TCO bit in uaccess_enable()
|
|
|
|
|
* and reset it in uaccess_disable().
|
|
|
|
|
*
|
|
|
|
|
* The Tag check override (TCO) bit disables temporarily the tag checking
|
|
|
|
|
* preventing the issue.
|
|
|
|
|
*/
|
2021-01-04 20:36:14 +08:00
|
|
|
static inline void __uaccess_disable_tco(void)
|
2016-07-01 21:58:21 +08:00
|
|
|
{
|
2020-12-23 04:01:35 +08:00
|
|
|
asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),
|
|
|
|
|
ARM64_MTE, CONFIG_KASAN_HW_TAGS));
|
2021-01-04 20:36:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void __uaccess_enable_tco(void)
|
|
|
|
|
{
|
|
|
|
|
asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),
|
|
|
|
|
ARM64_MTE, CONFIG_KASAN_HW_TAGS));
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-15 21:20:15 +08:00
|
|
|
/*
|
|
|
|
|
* These functions disable tag checking only if in MTE async mode
|
|
|
|
|
* since the sync mode generates exceptions synchronously and the
|
|
|
|
|
* nofault or load_unaligned_zeropad can handle them.
|
|
|
|
|
*/
|
|
|
|
|
static inline void __uaccess_disable_tco_async(void)
|
|
|
|
|
{
|
2021-10-06 23:47:50 +08:00
|
|
|
if (system_uses_mte_async_or_asymm_mode())
|
2021-03-15 21:20:15 +08:00
|
|
|
__uaccess_disable_tco();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void __uaccess_enable_tco_async(void)
|
|
|
|
|
{
|
2021-10-06 23:47:50 +08:00
|
|
|
if (system_uses_mte_async_or_asymm_mode())
|
2021-03-15 21:20:15 +08:00
|
|
|
__uaccess_enable_tco();
|
|
|
|
|
}
|
|
|
|
|
|
2021-01-04 20:36:14 +08:00
|
|
|
static inline void uaccess_disable_privileged(void)
|
|
|
|
|
{
|
|
|
|
|
__uaccess_disable_tco();
|
2020-12-23 04:01:35 +08:00
|
|
|
|
2020-12-02 21:15:57 +08:00
|
|
|
if (uaccess_ttbr0_disable())
|
|
|
|
|
return;
|
2016-07-01 21:58:21 +08:00
|
|
|
|
2020-12-02 21:15:57 +08:00
|
|
|
__uaccess_enable_hw_pan();
|
2016-07-01 21:58:21 +08:00
|
|
|
}
|
|
|
|
|
|
2020-12-02 21:15:57 +08:00
|
|
|
static inline void uaccess_enable_privileged(void)
|
2016-07-01 21:58:21 +08:00
|
|
|
{
|
2021-01-04 20:36:14 +08:00
|
|
|
__uaccess_enable_tco();
|
2020-12-23 04:01:35 +08:00
|
|
|
|
2020-12-02 21:15:57 +08:00
|
|
|
if (uaccess_ttbr0_enable())
|
|
|
|
|
return;
|
2016-07-01 21:58:21 +08:00
|
|
|
|
2020-12-02 21:15:57 +08:00
|
|
|
__uaccess_disable_hw_pan();
|
2016-07-01 21:58:21 +08:00
|
|
|
}
|
|
|
|
|
|
2018-02-05 23:34:19 +08:00
|
|
|
/*
|
arm64: uaccess: remove set_fs()
Now that the uaccess primitives dont take addr_limit into account, we
have no need to manipulate this via set_fs() and get_fs(). Remove
support for these, along with some infrastructure this renders
redundant.
We no longer need to flip UAO to access kernel memory under KERNEL_DS,
and head.S unconditionally clears UAO for all kernel configurations via
an ERET in init_kernel_el. Thus, we don't need to dynamically flip UAO,
nor do we need to context-switch it. However, we still need to adjust
PAN during SDEI entry.
Masking of __user pointers no longer needs to use the dynamic value of
addr_limit, and can use a constant derived from the maximum possible
userspace task size. A new TASK_SIZE_MAX constant is introduced for
this, which is also used by core code. In configurations supporting
52-bit VAs, this may include a region of unusable VA space above a
48-bit TTBR0 limit, but never includes any portion of TTBR1.
Note that TASK_SIZE_MAX is an exclusive limit, while USER_DS and
KERNEL_DS were inclusive limits, and is converted to a mask by
subtracting one.
As the SDEI entry code repurposes the otherwise unnecessary
pt_regs::orig_addr_limit field to store the TTBR1 of the interrupted
context, for now we rename that to pt_regs::sdei_ttbr1. In future we can
consider factoring that out.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: James Morse <james.morse@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201202131558.39270-10-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-12-02 21:15:55 +08:00
|
|
|
* Sanitise a uaccess pointer such that it becomes NULL if above the maximum
|
|
|
|
|
* user address. In case the pointer is tagged (has the top byte set), untag
|
|
|
|
|
* the pointer before checking.
|
2018-02-05 23:34:19 +08:00
|
|
|
*/
|
|
|
|
|
#define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
|
|
|
|
|
static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
|
|
|
|
|
{
|
|
|
|
|
void __user *safe_ptr;
|
|
|
|
|
|
|
|
|
|
asm volatile(
|
2019-07-24 01:58:38 +08:00
|
|
|
" bics xzr, %3, %2\n"
|
2018-02-05 23:34:19 +08:00
|
|
|
" csel %0, %1, xzr, eq\n"
|
|
|
|
|
: "=&r" (safe_ptr)
|
arm64: uaccess: remove set_fs()
Now that the uaccess primitives dont take addr_limit into account, we
have no need to manipulate this via set_fs() and get_fs(). Remove
support for these, along with some infrastructure this renders
redundant.
We no longer need to flip UAO to access kernel memory under KERNEL_DS,
and head.S unconditionally clears UAO for all kernel configurations via
an ERET in init_kernel_el. Thus, we don't need to dynamically flip UAO,
nor do we need to context-switch it. However, we still need to adjust
PAN during SDEI entry.
Masking of __user pointers no longer needs to use the dynamic value of
addr_limit, and can use a constant derived from the maximum possible
userspace task size. A new TASK_SIZE_MAX constant is introduced for
this, which is also used by core code. In configurations supporting
52-bit VAs, this may include a region of unusable VA space above a
48-bit TTBR0 limit, but never includes any portion of TTBR1.
Note that TASK_SIZE_MAX is an exclusive limit, while USER_DS and
KERNEL_DS were inclusive limits, and is converted to a mask by
subtracting one.
As the SDEI entry code repurposes the otherwise unnecessary
pt_regs::orig_addr_limit field to store the TTBR1 of the interrupted
context, for now we rename that to pt_regs::sdei_ttbr1. In future we can
consider factoring that out.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: James Morse <james.morse@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201202131558.39270-10-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-12-02 21:15:55 +08:00
|
|
|
: "r" (ptr), "r" (TASK_SIZE_MAX - 1),
|
2019-07-24 01:58:38 +08:00
|
|
|
"r" (untagged_addr(ptr))
|
2018-02-05 23:34:19 +08:00
|
|
|
: "cc");
|
|
|
|
|
|
|
|
|
|
csdb();
|
|
|
|
|
return safe_ptr;
|
|
|
|
|
}
|
|
|
|
|
|
2012-03-05 19:49:32 +08:00
|
|
|
/*
|
|
|
|
|
* The "__xxx" versions of the user access functions do not verify the address
|
|
|
|
|
* space - it must have been done previously with a separate "access_ok()"
|
|
|
|
|
* call.
|
|
|
|
|
*
|
|
|
|
|
* The "__xxx_error" versions set the third argument to -EFAULT if an error
|
|
|
|
|
* occurs, and leave it unchanged on success.
|
|
|
|
|
*/
|
2022-06-21 15:26:29 +08:00
|
|
|
#define __get_mem_asm(load, reg, x, addr, err, type) \
|
2012-03-05 19:49:32 +08:00
|
|
|
asm volatile( \
|
2020-12-02 21:15:53 +08:00
|
|
|
"1: " load " " reg "1, [%2]\n" \
|
2012-03-05 19:49:32 +08:00
|
|
|
"2:\n" \
|
2022-06-21 15:26:29 +08:00
|
|
|
_ASM_EXTABLE_##type##ACCESS_ERR_ZERO(1b, 2b, %w0, %w1) \
|
2012-03-05 19:49:32 +08:00
|
|
|
: "+r" (err), "=&r" (x) \
|
arm64: extable: add a dedicated uaccess handler
For inline assembly, we place exception fixups out-of-line in the
`.fixup` section such that these are out of the way of the fast path.
This has a few drawbacks:
* Since the fixup code is anonymous, backtraces will symbolize fixups as
offsets from the nearest prior symbol, currently
`__entry_tramp_text_end`. This is confusing, and painful to debug
without access to the relevant vmlinux.
* Since the exception handler adjusts the PC to execute the fixup, and
the fixup uses a direct branch back into the function it fixes,
backtraces of fixups miss the original function. This is confusing,
and violates requirements for RELIABLE_STACKTRACE (and therefore
LIVEPATCH).
* Inline assembly and associated fixups are generated from templates,
and we have many copies of logically identical fixups which only
differ in which specific registers are written to and which address is
branched to at the end of the fixup. This is potentially wasteful of
I-cache resources, and makes it hard to add additional logic to fixups
without significant bloat.
This patch address all three concerns for inline uaccess fixups by
adding a dedicated exception handler which updates registers in
exception context and subsequent returns back into the function which
faulted, removing the need for fixups specialized to each faulting
instruction.
Other than backtracing, there should be no functional change as a result
of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211019160219.5202-12-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-10-20 00:02:17 +08:00
|
|
|
: "r" (addr))
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2022-06-21 15:26:29 +08:00
|
|
|
#define __raw_get_mem(ldr, x, ptr, err, type) \
|
|
|
|
|
do { \
|
|
|
|
|
unsigned long __gu_val; \
|
|
|
|
|
switch (sizeof(*(ptr))) { \
|
|
|
|
|
case 1: \
|
|
|
|
|
__get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
case 2: \
|
|
|
|
|
__get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
case 4: \
|
|
|
|
|
__get_mem_asm(ldr, "%w", __gu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
case 8: \
|
|
|
|
|
__get_mem_asm(ldr, "%x", __gu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
default: \
|
|
|
|
|
BUILD_BUG(); \
|
|
|
|
|
} \
|
|
|
|
|
(x) = (__force __typeof__(*(ptr)))__gu_val; \
|
2018-09-06 19:09:56 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
/*
|
|
|
|
|
* We must not call into the scheduler between uaccess_ttbr0_enable() and
|
|
|
|
|
* uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions,
|
|
|
|
|
* we must evaluate these outside of the critical section.
|
|
|
|
|
*/
|
2020-12-02 21:15:52 +08:00
|
|
|
#define __raw_get_user(x, ptr, err) \
|
|
|
|
|
do { \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
__typeof__(*(ptr)) __user *__rgu_ptr = (ptr); \
|
|
|
|
|
__typeof__(x) __rgu_val; \
|
2020-12-02 21:15:52 +08:00
|
|
|
__chk_user_ptr(ptr); \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
\
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_enable(); \
|
2022-06-21 15:26:29 +08:00
|
|
|
__raw_get_mem("ldtr", __rgu_val, __rgu_ptr, err, U); \
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_disable(); \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
\
|
|
|
|
|
(x) = __rgu_val; \
|
2020-12-02 21:15:52 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
2019-01-15 21:58:26 +08:00
|
|
|
#define __get_user_error(x, ptr, err) \
|
|
|
|
|
do { \
|
2018-02-05 23:34:22 +08:00
|
|
|
__typeof__(*(ptr)) __user *__p = (ptr); \
|
|
|
|
|
might_fault(); \
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 10:57:57 +08:00
|
|
|
if (access_ok(__p, sizeof(*__p))) { \
|
2018-02-05 23:34:22 +08:00
|
|
|
__p = uaccess_mask_ptr(__p); \
|
2019-03-01 22:19:06 +08:00
|
|
|
__raw_get_user((x), __p, (err)); \
|
2018-02-05 23:34:22 +08:00
|
|
|
} else { \
|
2020-05-22 22:23:21 +08:00
|
|
|
(x) = (__force __typeof__(x))0; (err) = -EFAULT; \
|
2018-02-05 23:34:22 +08:00
|
|
|
} \
|
2019-01-15 21:58:26 +08:00
|
|
|
} while (0)
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2018-02-05 23:34:22 +08:00
|
|
|
#define __get_user(x, ptr) \
|
2012-03-05 19:49:32 +08:00
|
|
|
({ \
|
2018-02-05 23:34:22 +08:00
|
|
|
int __gu_err = 0; \
|
2019-01-15 21:58:26 +08:00
|
|
|
__get_user_error((x), (ptr), __gu_err); \
|
2018-02-05 23:34:22 +08:00
|
|
|
__gu_err; \
|
2012-03-05 19:49:32 +08:00
|
|
|
})
|
|
|
|
|
|
2018-02-05 23:34:22 +08:00
|
|
|
#define get_user __get_user
|
|
|
|
|
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
/*
|
|
|
|
|
* We must not call into the scheduler between __uaccess_enable_tco_async() and
|
|
|
|
|
* __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking
|
|
|
|
|
* functions, we must evaluate these outside of the critical section.
|
|
|
|
|
*/
|
2020-12-02 21:15:53 +08:00
|
|
|
#define __get_kernel_nofault(dst, src, type, err_label) \
|
|
|
|
|
do { \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
__typeof__(dst) __gkn_dst = (dst); \
|
|
|
|
|
__typeof__(src) __gkn_src = (src); \
|
2020-12-02 21:15:53 +08:00
|
|
|
int __gkn_err = 0; \
|
|
|
|
|
\
|
2021-03-15 21:20:15 +08:00
|
|
|
__uaccess_enable_tco_async(); \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
__raw_get_mem("ldr", *((type *)(__gkn_dst)), \
|
2022-06-21 15:26:29 +08:00
|
|
|
(__force type *)(__gkn_src), __gkn_err, K); \
|
2021-03-15 21:20:15 +08:00
|
|
|
__uaccess_disable_tco_async(); \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
\
|
2020-12-02 21:15:53 +08:00
|
|
|
if (unlikely(__gkn_err)) \
|
|
|
|
|
goto err_label; \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
2022-06-21 15:26:29 +08:00
|
|
|
#define __put_mem_asm(store, reg, x, addr, err, type) \
|
2012-03-05 19:49:32 +08:00
|
|
|
asm volatile( \
|
2020-12-02 21:15:53 +08:00
|
|
|
"1: " store " " reg "1, [%2]\n" \
|
2012-03-05 19:49:32 +08:00
|
|
|
"2:\n" \
|
2022-06-21 15:26:29 +08:00
|
|
|
_ASM_EXTABLE_##type##ACCESS_ERR(1b, 2b, %w0) \
|
2012-03-05 19:49:32 +08:00
|
|
|
: "+r" (err) \
|
arm64: extable: add a dedicated uaccess handler
For inline assembly, we place exception fixups out-of-line in the
`.fixup` section such that these are out of the way of the fast path.
This has a few drawbacks:
* Since the fixup code is anonymous, backtraces will symbolize fixups as
offsets from the nearest prior symbol, currently
`__entry_tramp_text_end`. This is confusing, and painful to debug
without access to the relevant vmlinux.
* Since the exception handler adjusts the PC to execute the fixup, and
the fixup uses a direct branch back into the function it fixes,
backtraces of fixups miss the original function. This is confusing,
and violates requirements for RELIABLE_STACKTRACE (and therefore
LIVEPATCH).
* Inline assembly and associated fixups are generated from templates,
and we have many copies of logically identical fixups which only
differ in which specific registers are written to and which address is
branched to at the end of the fixup. This is potentially wasteful of
I-cache resources, and makes it hard to add additional logic to fixups
without significant bloat.
This patch address all three concerns for inline uaccess fixups by
adding a dedicated exception handler which updates registers in
exception context and subsequent returns back into the function which
faulted, removing the need for fixups specialized to each faulting
instruction.
Other than backtracing, there should be no functional change as a result
of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20211019160219.5202-12-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-10-20 00:02:17 +08:00
|
|
|
: "r" (x), "r" (addr))
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2022-06-21 15:26:29 +08:00
|
|
|
#define __raw_put_mem(str, x, ptr, err, type) \
|
|
|
|
|
do { \
|
|
|
|
|
__typeof__(*(ptr)) __pu_val = (x); \
|
|
|
|
|
switch (sizeof(*(ptr))) { \
|
|
|
|
|
case 1: \
|
|
|
|
|
__put_mem_asm(str "b", "%w", __pu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
case 2: \
|
|
|
|
|
__put_mem_asm(str "h", "%w", __pu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
case 4: \
|
|
|
|
|
__put_mem_asm(str, "%w", __pu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
case 8: \
|
|
|
|
|
__put_mem_asm(str, "%x", __pu_val, (ptr), (err), type); \
|
|
|
|
|
break; \
|
|
|
|
|
default: \
|
|
|
|
|
BUILD_BUG(); \
|
|
|
|
|
} \
|
2020-12-02 21:15:52 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
/*
|
|
|
|
|
* We must not call into the scheduler between uaccess_ttbr0_enable() and
|
|
|
|
|
* uaccess_ttbr0_disable(). As `x` and `ptr` could contain blocking functions,
|
|
|
|
|
* we must evaluate these outside of the critical section.
|
|
|
|
|
*/
|
2020-12-02 21:15:52 +08:00
|
|
|
#define __raw_put_user(x, ptr, err) \
|
|
|
|
|
do { \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
__typeof__(*(ptr)) __user *__rpu_ptr = (ptr); \
|
|
|
|
|
__typeof__(*(ptr)) __rpu_val = (x); \
|
|
|
|
|
__chk_user_ptr(__rpu_ptr); \
|
|
|
|
|
\
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_enable(); \
|
2022-06-21 15:26:29 +08:00
|
|
|
__raw_put_mem("sttr", __rpu_val, __rpu_ptr, err, U); \
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_disable(); \
|
2012-03-05 19:49:32 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
2019-01-15 21:58:26 +08:00
|
|
|
#define __put_user_error(x, ptr, err) \
|
|
|
|
|
do { \
|
2018-02-05 23:34:22 +08:00
|
|
|
__typeof__(*(ptr)) __user *__p = (ptr); \
|
|
|
|
|
might_fault(); \
|
Remove 'type' argument from access_ok() function
Nobody has actually used the type (VERIFY_READ vs VERIFY_WRITE) argument
of the user address range verification function since we got rid of the
old racy i386-only code to walk page tables by hand.
It existed because the original 80386 would not honor the write protect
bit when in kernel mode, so you had to do COW by hand before doing any
user access. But we haven't supported that in a long time, and these
days the 'type' argument is a purely historical artifact.
A discussion about extending 'user_access_begin()' to do the range
checking resulted this patch, because there is no way we're going to
move the old VERIFY_xyz interface to that model. And it's best done at
the end of the merge window when I've done most of my merges, so let's
just get this done once and for all.
This patch was mostly done with a sed-script, with manual fix-ups for
the cases that weren't of the trivial 'access_ok(VERIFY_xyz' form.
There were a couple of notable cases:
- csky still had the old "verify_area()" name as an alias.
- the iter_iov code had magical hardcoded knowledge of the actual
values of VERIFY_{READ,WRITE} (not that they mattered, since nothing
really used it)
- microblaze used the type argument for a debug printout
but other than those oddities this should be a total no-op patch.
I tried to fix up all architectures, did fairly extensive grepping for
access_ok() uses, and the changes are trivial, but I may have missed
something. Any missed conversion should be trivially fixable, though.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2019-01-04 10:57:57 +08:00
|
|
|
if (access_ok(__p, sizeof(*__p))) { \
|
2018-02-05 23:34:22 +08:00
|
|
|
__p = uaccess_mask_ptr(__p); \
|
2019-03-01 22:19:06 +08:00
|
|
|
__raw_put_user((x), __p, (err)); \
|
2018-02-05 23:34:22 +08:00
|
|
|
} else { \
|
|
|
|
|
(err) = -EFAULT; \
|
|
|
|
|
} \
|
2019-01-15 21:58:26 +08:00
|
|
|
} while (0)
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2018-02-05 23:34:22 +08:00
|
|
|
#define __put_user(x, ptr) \
|
2012-03-05 19:49:32 +08:00
|
|
|
({ \
|
2018-02-05 23:34:22 +08:00
|
|
|
int __pu_err = 0; \
|
2019-01-15 21:58:26 +08:00
|
|
|
__put_user_error((x), (ptr), __pu_err); \
|
2018-02-05 23:34:22 +08:00
|
|
|
__pu_err; \
|
2012-03-05 19:49:32 +08:00
|
|
|
})
|
|
|
|
|
|
2018-02-05 23:34:22 +08:00
|
|
|
#define put_user __put_user
|
|
|
|
|
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
/*
|
|
|
|
|
* We must not call into the scheduler between __uaccess_enable_tco_async() and
|
|
|
|
|
* __uaccess_disable_tco_async(). As `dst` and `src` may contain blocking
|
|
|
|
|
* functions, we must evaluate these outside of the critical section.
|
|
|
|
|
*/
|
2020-12-02 21:15:53 +08:00
|
|
|
#define __put_kernel_nofault(dst, src, type, err_label) \
|
|
|
|
|
do { \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
__typeof__(dst) __pkn_dst = (dst); \
|
|
|
|
|
__typeof__(src) __pkn_src = (src); \
|
2020-12-02 21:15:53 +08:00
|
|
|
int __pkn_err = 0; \
|
|
|
|
|
\
|
2021-03-15 21:20:15 +08:00
|
|
|
__uaccess_enable_tco_async(); \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
__raw_put_mem("str", *((type *)(__pkn_src)), \
|
2022-06-21 15:26:29 +08:00
|
|
|
(__force type *)(__pkn_dst), __pkn_err, K); \
|
2021-03-15 21:20:15 +08:00
|
|
|
__uaccess_disable_tco_async(); \
|
arm64: uaccess: avoid blocking within critical sections
As Vincent reports in:
https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
The put_user() in schedule_tail() can get stuck in a livelock, similar
to a problem recently fixed on riscv in commit:
285a76bb2cf51b0c ("riscv: evaluate put_user() arg before enabling user access")
In __raw_put_user() we have a critical section between
uaccess_ttbr0_enable() and uaccess_ttbr0_disable() where we cannot
safely call into the scheduler without having taken an exception, as
schedule() and other scheduling functions will not save/restore the
TTBR0 state. If either of the `x` or `ptr` arguments to __raw_put_user()
contain a blocking call, we may call into the scheduler within the
critical section. This can result in two problems:
1) The access within the critical section will occur without the
required TTBR0 tables installed. This will fault, and where the
required tables permit access, the access will be retried without the
required tables, resulting in a livelock.
2) When TTBR0 SW PAN is in use, check_and_switch_context() does not
modify TTBR0, leaving a stale value installed. The mappings of the
blocked task will erroneously be accessible to regular accesses in
the context of the new task. Additionally, if the tables are
subsequently freed, local TLB maintenance required to reuse the ASID
may be lost, potentially resulting in TLB corruption (e.g. in the
presence of CnP).
The same issue exists for __raw_get_user() in the critical section
between uaccess_ttbr0_enable() and uaccess_ttbr0_disable().
A similar issue exists for __get_kernel_nofault() and
__put_kernel_nofault() for the critical section between
__uaccess_enable_tco_async() and __uaccess_disable_tco_async(), as the
TCO state is not context-switched by direct calls into the scheduler.
Here the TCO state may be lost from the context of the current task,
resulting in unexpected asynchronous tag check faults. It may also be
leaked to another task, suppressing expected tag check faults.
To fix all of these cases, we must ensure that we do not directly call
into the scheduler in their respective critical sections. This patch
reworks __raw_put_user(), __raw_get_user(), __get_kernel_nofault(), and
__put_kernel_nofault(), ensuring that parameters are evaluated outside
of the critical sections. To make this requirement clear, comments are
added describing the problem, and line spaces added to separate the
critical sections from other portions of the macros.
For __raw_get_user() and __raw_put_user() the `err` parameter is
conditionally assigned to, and we must currently evaluate this in the
critical section. This behaviour is relied upon by the signal code,
which uses chains of put_user_error() and get_user_error(), checking the
return value at the end. In all cases, the `err` parameter is a plain
int rather than a more complex expression with a blocking call, so this
is safe.
In future we should try to clean up the `err` usage to remove the
potential for this to be a problem.
Aside from the changes to time of evaluation, there should be no
functional change as a result of this patch.
Reported-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
Link: https://lore.kernel.org/r/20211118163417.21617-1-vincent.whitchurch@axis.com
Fixes: f253d827f33c ("arm64: uaccess: refactor __{get,put}_user")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20211122125820.55286-1-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>
2021-11-22 20:58:20 +08:00
|
|
|
\
|
2020-12-02 21:15:53 +08:00
|
|
|
if (unlikely(__pkn_err)) \
|
|
|
|
|
goto err_label; \
|
|
|
|
|
} while(0)
|
|
|
|
|
|
2016-06-09 05:40:56 +08:00
|
|
|
extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
|
2018-02-05 23:34:23 +08:00
|
|
|
#define raw_copy_from_user(to, from, n) \
|
|
|
|
|
({ \
|
2019-11-21 01:07:40 +08:00
|
|
|
unsigned long __acfu_ret; \
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_enable(); \
|
2019-11-21 01:07:40 +08:00
|
|
|
__acfu_ret = __arch_copy_from_user((to), \
|
|
|
|
|
__uaccess_mask_ptr(from), (n)); \
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_disable(); \
|
2019-11-21 01:07:40 +08:00
|
|
|
__acfu_ret; \
|
2018-02-05 23:34:23 +08:00
|
|
|
})
|
|
|
|
|
|
2016-06-09 05:40:56 +08:00
|
|
|
extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
|
2018-02-05 23:34:23 +08:00
|
|
|
#define raw_copy_to_user(to, from, n) \
|
|
|
|
|
({ \
|
2019-11-21 01:07:40 +08:00
|
|
|
unsigned long __actu_ret; \
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_enable(); \
|
2019-11-21 01:07:40 +08:00
|
|
|
__actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \
|
|
|
|
|
(from), (n)); \
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_disable(); \
|
2019-11-21 01:07:40 +08:00
|
|
|
__actu_ret; \
|
2018-02-05 23:34:23 +08:00
|
|
|
})
|
|
|
|
|
|
2017-03-21 20:40:57 +08:00
|
|
|
#define INLINE_COPY_TO_USER
|
|
|
|
|
#define INLINE_COPY_FROM_USER
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2018-02-05 23:34:23 +08:00
|
|
|
extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
|
|
|
|
|
static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
|
2012-03-05 19:49:32 +08:00
|
|
|
{
|
2019-11-21 01:07:40 +08:00
|
|
|
if (access_ok(to, n)) {
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_enable();
|
2018-02-05 23:34:23 +08:00
|
|
|
n = __arch_clear_user(__uaccess_mask_ptr(to), n);
|
2020-12-02 21:15:57 +08:00
|
|
|
uaccess_ttbr0_disable();
|
2019-11-21 01:07:40 +08:00
|
|
|
}
|
2012-03-05 19:49:32 +08:00
|
|
|
return n;
|
|
|
|
|
}
|
2018-02-05 23:34:23 +08:00
|
|
|
#define clear_user __clear_user
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2013-11-07 01:20:22 +08:00
|
|
|
extern long strncpy_from_user(char *dest, const char __user *src, long count);
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2013-11-07 01:20:22 +08:00
|
|
|
extern __must_check long strnlen_user(const char __user *str, long n);
|
2012-03-05 19:49:32 +08:00
|
|
|
|
2017-07-25 18:55:43 +08:00
|
|
|
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
|
|
|
|
|
struct page;
|
|
|
|
|
void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
|
|
|
|
|
extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
|
|
|
|
|
|
|
|
|
|
static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
|
|
|
|
|
{
|
|
|
|
|
kasan_check_write(dst, size);
|
2018-02-05 23:34:23 +08:00
|
|
|
return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
|
2017-07-25 18:55:43 +08:00
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2022-04-23 18:07:50 +08:00
|
|
|
#ifdef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Return 0 on success, the number of bytes not probed otherwise.
|
|
|
|
|
*/
|
|
|
|
|
static inline size_t probe_subpage_writeable(const char __user *uaddr,
|
|
|
|
|
size_t size)
|
|
|
|
|
{
|
|
|
|
|
if (!system_supports_mte())
|
|
|
|
|
return 0;
|
|
|
|
|
return mte_probe_user_range(uaddr, size);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */
|
|
|
|
|
|
2012-03-05 19:49:32 +08:00
|
|
|
#endif /* __ASM_UACCESS_H */
|