UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

ARM: smp: Store current pointer in TPIDRURO register if available 

Now that the user space TLS register is assigned on every return to user
space, we can use it to keep the 'current' pointer while running in the
kernel. This removes the need to access it via thread_info, which is
located at the base of the stack, but will be moved out of there in a
subsequent patch.

Use the __builtin_thread_pointer() helper when available - this will
help GCC understand that reloading the value within the same function is
not necessary, even when using the per-task stack protector (which also
generates accesses via the TLS register). For example, the generated
code below loads TPIDRURO only once, and uses it to access both the
stack canary and the preempt_count fields.

<do_one_initcall>:
       e92d 41f0       stmdb   sp!, {r4, r5, r6, r7, r8, lr}
       ee1d 4f70       mrc     15, 0, r4, cr13, cr0, {3}
       4606            mov     r6, r0
       b094            sub     sp, #80 ; 0x50
       f8d4 34e8       ldr.w   r3, [r4, #1256] ; 0x4e8  <- stack canary
       9313            str     r3, [sp, #76]   ; 0x4c
       f8d4 8004       ldr.w   r8, [r4, #4]             <- preempt count

Co-developed-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Linus Walleij <linus.walleij@linaro.org>
Tested-by: Amit Daniel Kachhap <amit.kachhap@arm.com>
This commit is contained in:
Ard Biesheuvel 2021-09-18 10:44:37 +02:00
parent 3855ab614d
commit 50596b7559
12 changed files with 105 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
arch/arm/Kconfig
View File

@ -1157,6 +1157,11 @@ config SMP_ON_UP
If you don't know what to do here, say Y.
config CURRENT_POINTER_IN_TPIDRURO
def_bool y
depends on SMP && CPU_32v6K && !CPU_V6
config ARM_CPU_TOPOLOGY
bool "Support cpu topology definition"
depends on SMP && CPU_V7

4
arch/arm/Makefile
View File

@ -113,6 +113,10 @@ ifeq ($(CONFIG_CC_IS_CLANG),y)
CFLAGS_ABI += -meabi gnu
endif
ifeq ($(CONFIG_CURRENT_POINTER_IN_TPIDRURO),y)
CFLAGS_ABI += -mtp=cp15
endif
# Accept old syntax despite ".syntax unified"
AFLAGS_NOWARN :=$(call as-option,-Wa$(comma)-mno-warn-deprecated,-Wa$(comma)-W)

24
arch/arm/include/asm/assembler.h
View File

@ -199,6 +199,30 @@
.endm
.endr
.macro get_current, rd
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register
#else
get_thread_info \rd
ldr \rd, [\rd, #TI_TASK]
#endif
.endm
.macro set_current, rn
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register
#endif
.endm
.macro reload_current, t1:req, t2:req
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
adr_l \t1, __entry_task @ get __entry_task base address
mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset
ldr \t1, [\t1, \t2] @ load variable
mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
#endif
.endm
/*
* Get current thread_info.
*/

50
arch/arm/include/asm/current.h Normal file
View File

@ -0,0 +1,50 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2021 Keith Packard <keithp@keithp.com>
* Copyright (c) 2021 Google, LLC <ardb@kernel.org>
*/
#ifndef _ASM_ARM_CURRENT_H
#define _ASM_ARM_CURRENT_H
#ifndef __ASSEMBLY__
struct task_struct;
static inline void set_current(struct task_struct *cur)
{
if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO))
return;
/* Set TPIDRURO */
asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");
}
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
static inline struct task_struct *get_current(void)
{
struct task_struct *cur;
#if __has_builtin(__builtin_thread_pointer)
/*
* Use the __builtin helper when available - this results in better
* code, especially when using GCC in combination with the per-task
* stack protector, as the compiler will recognize that it needs to
* load the TLS register only once in every function.
*/
cur = __builtin_thread_pointer();
#else
asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur));
#endif
return cur;
}
#define current get_current()
#else
#include <asm-generic/current.h>
#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */
#endif /* __ASSEMBLY__ */
#endif /* _ASM_ARM_CURRENT_H */

2
arch/arm/include/asm/switch_to.h
View File

@ -26,6 +26,8 @@ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info
#define switch_to(prev,next,last) \
do { \
__complete_pending_tlbi(); \
if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \
__this_cpu_write(__entry_task, next); \
last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
} while (0)

2
arch/arm/include/asm/thread_info.h
View File

@ -29,6 +29,8 @@
struct task_struct;
DECLARE_PER_CPU(struct task_struct *, __entry_task);
#include <asm/types.h>
struct cpu_context_save {

5
arch/arm/kernel/entry-armv.S
View File

@ -384,6 +384,8 @@ ENDPROC(__fiq_abt)
ATRAP( teq r8, r7)
ATRAP( mcrne p15, 0, r8, c1, c0, 0)
reload_current r7, r8
@
@ Clear FP to mark the first stack frame
@
@ -762,6 +764,8 @@ ENTRY(__switch_to)
add r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK
.endif
ldr r7, [r7, #TSK_STACK_CANARY & IMM12_MASK]
#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO)
ldr r7, [r2, #TI_TASK]
#endif
#ifdef CONFIG_CPU_USE_DOMAINS
mcr p15, 0, r6, c3, c0, 0 @ Set domain register
@ -776,6 +780,7 @@ ENTRY(__switch_to)
#endif
THUMB( mov ip, r4 )
mov r0, r5
set_current r7
ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously
THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously
THUMB( ldr sp, [ip], #4 )

1
arch/arm/kernel/entry-common.S
View File

@ -170,6 +170,7 @@ ENTRY(vector_swi)
str saved_psr, [sp, #S_PSR] @ Save CPSR
str r0, [sp, #S_OLD_R0] @ Save OLD_R0
#endif
reload_current r10, ip
zero_fp
alignment_trap r10, ip, __cr_alignment
asm_trace_hardirqs_on save=0

5
arch/arm/kernel/head-common.S
View File

@ -105,6 +105,11 @@ __mmap_switched:
mov r1, #0
bl __memset @ clear .bss
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
adr_l r0, init_task @ get swapper task_struct
set_current r0
#endif
ldmia r4, {r0, r1, r2, r3}
str r9, [r0] @ Save processor ID
str r7, [r1] @ Save machine type

4
arch/arm/kernel/process.c
View File

@ -36,6 +36,10 @@
#include "signal.h"
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
DEFINE_PER_CPU(struct task_struct *, __entry_task);
#endif
#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
#include <linux/stackprotector.h>
unsigned long __stack_chk_guard __read_mostly;

2
arch/arm/kernel/smp.c
View File

@ -409,6 +409,8 @@ asmlinkage void secondary_start_kernel(struct task_struct *task)
struct mm_struct *mm = &init_mm;
unsigned int cpu;
set_current(task);
secondary_biglittle_init();
/*

3
arch/arm/mm/proc-macros.S
View File

@ -30,8 +30,7 @@
* act_mm - get current->active_mm
*/
.macro act_mm, rd
get_thread_info \rd
ldr \rd, [\rd, #TI_TASK]
get_current \rd
.if (TSK_ACTIVE_MM > IMM12_MASK)
add \rd, \rd, #TSK_ACTIVE_MM & ~IMM12_MASK
.endif