UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

ARM: support for IRQ and vmap'ed stacks [v6] 

This tag covers the changes between the version of vmap'ed + IRQ stacks
 support pulled into rmk/devel-stable [0] (which was dropped from v5.17
 due to issues discovered too late in the cycle), and my v5 proposed for
 the v5.18 cycle [1].
 
 [0] git://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git arm-irq-and-vmap-stacks-for-rmk
 [1] https://lore.kernel.org/linux-arm-kernel/20220124174744.1054712-1-ardb@kernel.org/
 -----BEGIN PGP SIGNATURE-----
 
 iQGzBAABCgAdFiEE+9lifEBpyUIVN1cpw08iOZLZjyQFAmH3+1oACgkQw08iOZLZ
 jyRdyAv/TiYdEkpteCUz1MucDFEZsRz1FXYTUwFG5pxSIONUDdDm0KvjYoY80n7X
 wUMZyfAwjdHpQtP0iu4RwAmi7d373KtWTqFzwAoBG9RFTSy/4j4B3ZzsPkoCn9uN
 ANXpyJE2lqvN3d25WKnRq6+WGSxdvhYqBQARe1oznirgN4ilKtmBkKCL3W+gsO7l
 N6q5DLsqSI80kAIorFUr0sF8b1JEK/APOokaAICLyP6fkjp3hu+jUvJENCsJk27V
 rVHhFmKdtpwl02hs+I13I5nrAXwYN6COSBa9y0xuPRgBk2sgnpFKSMKAvYafwHhg
 AYwUuez/Tk6AHHowu+/ggoap2At04l4rdwzV0BIE/+9vdT3C+4M5tikHglQnRjtR
 PRyErdCPPEW6gz+fYdYoaCYXVfRGCQeCyInVQIl6U9HAqcVLPHNZecGz0rYBTQA2
 GiUfi0YA3SASMIggP4mug4M5fwbgUbh/i3OgMYGcnCg+5phmR7Z+niJVN9j0uPf2
 XMsCsTi/
 =utwu
 -----END PGP SIGNATURE-----

Merge tag 'arm-vmap-stacks-v6' of git://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux into devel-stable

ARM: support for IRQ and vmap'ed stacks [v6]

This tag covers the changes between the version of vmap'ed + IRQ stacks
support pulled into rmk/devel-stable [0] (which was dropped from v5.17
due to issues discovered too late in the cycle), and my v5 proposed for
the v5.18 cycle [1].

[0] git://git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git arm-irq-and-vmap-stacks-for-rmk
[1] https://lore.kernel.org/linux-arm-kernel/20220124174744.1054712-1-ardb@kernel.org/
This commit is contained in:
Russell King (Oracle) 2022-01-31 15:26:45 +00:00
parent 5fe41793bc 4d5a643e73
commit 2fa3948244
19 changed files with 115 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
arch/arm/Kconfig
View File

@ -128,7 +128,7 @@ config ARM
select RTC_LIB select RTC_LIB
select SYS_SUPPORTS_APM_EMULATION select SYS_SUPPORTS_APM_EMULATION
select THREAD_INFO_IN_TASK select THREAD_INFO_IN_TASK
select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) && !PM_SLEEP_SMP select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS
select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M
# Above selects are sorted alphabetically; please add new ones # Above selects are sorted alphabetically; please add new ones
# according to that. Thanks. # according to that. Thanks.
@ -140,6 +140,17 @@ config ARM
Europe. There is an ARM Linux project with a web page at Europe. There is an ARM Linux project with a web page at
<http://www.arm.linux.org.uk/>. <http://www.arm.linux.org.uk/>.
config ARM_HAS_GROUP_RELOCS
def_bool y
depends on !LD_IS_LLD || LLD_VERSION >= 140000
depends on !COMPILE_TEST
help
Whether or not to use R_ARM_ALU_PC_Gn or R_ARM_LDR_PC_Gn group
relocations, which have been around for a long time, but were not
supported in LLD until version 14. The combined range is -/+ 256 MiB,
which is usually sufficient, but not for allyesconfig, so we disable
this feature when doing compile testing.
config ARM_HAS_SG_CHAIN config ARM_HAS_SG_CHAIN
bool bool

8
arch/arm/include/asm/assembler.h
View File

@ -656,8 +656,8 @@ THUMB( orr \reg , \reg , #PSR_T_BIT )
.macro __ldst_va, op, reg, tmp, sym, cond .macro __ldst_va, op, reg, tmp, sym, cond
#if __LINUX_ARM_ARCH__ >= 7 || \ #if __LINUX_ARM_ARCH__ >= 7 || \
(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
mov_l \tmp, \sym, \cond mov_l \tmp, \sym, \cond
\op\cond \reg, [\tmp] \op\cond \reg, [\tmp]
#else #else
@ -716,8 +716,8 @@ THUMB( orr \reg , \reg , #PSR_T_BIT )
*/ */
.macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req
#if __LINUX_ARM_ARCH__ >= 7 || \ #if __LINUX_ARM_ARCH__ >= 7 || \
(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
this_cpu_offset \t1 this_cpu_offset \t1
mov_l \t2, \sym mov_l \t2, \sym
ldr \rd, [\t1, \t2] ldr \rd, [\t1, \t2]

10
arch/arm/include/asm/current.h
View File

@ -14,7 +14,7 @@ struct task_struct;
extern struct task_struct *__current; extern struct task_struct *__current;
static inline __attribute_const__ struct task_struct *get_current(void) static __always_inline __attribute_const__ struct task_struct *get_current(void)
{ {
struct task_struct *cur; struct task_struct *cur;
@ -37,8 +37,8 @@ static inline __attribute_const__ struct task_struct *get_current(void)
#ifdef CONFIG_CPU_V6 #ifdef CONFIG_CPU_V6
"1: \n\t" "1: \n\t"
" .subsection 1 \n\t" " .subsection 1 \n\t"
#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ #if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \
!(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
"2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t"
" b 1b \n\t" " b 1b \n\t"
#else #else
@ -55,8 +55,8 @@ static inline __attribute_const__ struct task_struct *get_current(void)
#endif #endif
: "=r"(cur)); : "=r"(cur));
#elif __LINUX_ARM_ARCH__>= 7 || \ #elif __LINUX_ARM_ARCH__>= 7 || \
(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \
(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
cur = __current; cur = __current;
#else #else
asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur));

2
arch/arm/include/asm/mmu.h
View File

@ -10,7 +10,7 @@ typedef struct {
#else #else
int switch_pending; int switch_pending;
#endif #endif
unsigned int vmalloc_seq; atomic_t vmalloc_seq;
unsigned long sigpage; unsigned long sigpage;
#ifdef CONFIG_VDSO #ifdef CONFIG_VDSO
unsigned long vdso; unsigned long vdso;

22
arch/arm/include/asm/mmu_context.h
View File

@ -23,6 +23,16 @@
void __check_vmalloc_seq(struct mm_struct *mm); void __check_vmalloc_seq(struct mm_struct *mm);
#ifdef CONFIG_MMU
static inline void check_vmalloc_seq(struct mm_struct *mm)
{
if (!IS_ENABLED(CONFIG_ARM_LPAE) &&
unlikely(atomic_read(&mm->context.vmalloc_seq) !=
atomic_read(&init_mm.context.vmalloc_seq)))
__check_vmalloc_seq(mm);
}
#endif
#ifdef CONFIG_CPU_HAS_ASID #ifdef CONFIG_CPU_HAS_ASID
void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk); void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk);
@ -52,8 +62,7 @@ static inline void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
static inline void check_and_switch_context(struct mm_struct *mm, static inline void check_and_switch_context(struct mm_struct *mm,
struct task_struct *tsk) struct task_struct *tsk)
{ {
if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) check_vmalloc_seq(mm);
__check_vmalloc_seq(mm);
if (irqs_disabled()) if (irqs_disabled())
/* /*
@ -129,6 +138,15 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
#endif #endif
} }
#ifdef CONFIG_VMAP_STACK
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
if (mm != &init_mm)
check_vmalloc_seq(mm);
}
#define enter_lazy_tlb enter_lazy_tlb
#endif
#include <asm-generic/mmu_context.h> #include <asm-generic/mmu_context.h>
#endif #endif

3
arch/arm/include/asm/page.h
View File

@ -147,11 +147,10 @@ extern void copy_page(void *to, const void *from);
#include <asm/pgtable-3level-types.h> #include <asm/pgtable-3level-types.h>
#else #else
#include <asm/pgtable-2level-types.h> #include <asm/pgtable-2level-types.h>
#endif
#ifdef CONFIG_VMAP_STACK #ifdef CONFIG_VMAP_STACK
#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED #define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED
#endif #endif
#endif
#endif /* CONFIG_MMU */ #endif /* CONFIG_MMU */

6
arch/arm/include/asm/percpu.h
View File

@ -25,7 +25,7 @@ static inline void set_my_cpu_offset(unsigned long off)
asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory"); asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory");
} }
static inline unsigned long __my_cpu_offset(void) static __always_inline unsigned long __my_cpu_offset(void)
{ {
unsigned long off; unsigned long off;
@ -38,8 +38,8 @@ static inline unsigned long __my_cpu_offset(void)
#ifdef CONFIG_CPU_V6 #ifdef CONFIG_CPU_V6
"1: \n\t" "1: \n\t"
" .subsection 1 \n\t" " .subsection 1 \n\t"
#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ #if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \
!(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS))
"2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t" "2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t"
" b 1b \n\t" " b 1b \n\t"
#else #else

4
arch/arm/include/asm/switch_to.h
View File

@ -3,6 +3,7 @@
#define __ASM_ARM_SWITCH_TO_H #define __ASM_ARM_SWITCH_TO_H
#include <linux/thread_info.h> #include <linux/thread_info.h>
#include <asm/smp_plat.h>
/* /*
* For v7 SMP cores running a preemptible kernel we may be pre-empted * For v7 SMP cores running a preemptible kernel we may be pre-empted
@ -40,8 +41,7 @@ static inline void set_ti_cpu(struct task_struct *p)
do { \ do { \
__complete_pending_tlbi(); \ __complete_pending_tlbi(); \
set_ti_cpu(next); \ set_ti_cpu(next); \
if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || \ if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || is_smp()) \
IS_ENABLED(CONFIG_SMP)) \
__this_cpu_write(__entry_task, next); \ __this_cpu_write(__entry_task, next); \
last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
} while (0) } while (0)

22
arch/arm/include/asm/tls.h
View File

@ -18,22 +18,32 @@
.endm .endm
.macro switch_tls_v6, base, tp, tpuser, tmp1, tmp2 .macro switch_tls_v6, base, tp, tpuser, tmp1, tmp2
#ifdef CONFIG_SMP
ALT_SMP(nop)
ALT_UP_B(.L0_\@)
.subsection 1
#endif
.L0_\@:
ldr_va \tmp1, elf_hwcap ldr_va \tmp1, elf_hwcap
mov \tmp2, #0xffff0fff mov \tmp2, #0xffff0fff
tst \tmp1, #HWCAP_TLS @ hardware TLS available? tst \tmp1, #HWCAP_TLS @ hardware TLS available?
streq \tp, [\tmp2, #-15] @ set TLS value at 0xffff0ff0 streq \tp, [\tmp2, #-15] @ set TLS value at 0xffff0ff0
mrcne p15, 0, \tmp2, c13, c0, 2 @ get the user r/w register beq .L2_\@
#ifndef CONFIG_SMP mcr p15, 0, \tp, c13, c0, 3 @ yes, set TLS register
mcrne p15, 0, \tp, c13, c0, 3 @ yes, set TLS register #ifdef CONFIG_SMP
b .L1_\@
.previous
#endif #endif
mcrne p15, 0, \tpuser, c13, c0, 2 @ set user r/w register .L1_\@: switch_tls_v6k \base, \tp, \tpuser, \tmp1, \tmp2
strne \tmp2, [\base, #TI_TP_VALUE + 4] @ save it .L2_\@:
.endm .endm
.macro switch_tls_software, base, tp, tpuser, tmp1, tmp2 .macro switch_tls_software, base, tp, tpuser, tmp1, tmp2
mov \tmp1, #0xffff0fff mov \tmp1, #0xffff0fff
str \tp, [\tmp1, #-15] @ set TLS value at 0xffff0ff0 str \tp, [\tmp1, #-15] @ set TLS value at 0xffff0ff0
.endm .endm
#else
#include <asm/smp_plat.h>
#endif #endif
#ifdef CONFIG_TLS_REG_EMUL #ifdef CONFIG_TLS_REG_EMUL
@ -44,7 +54,7 @@
#elif defined(CONFIG_CPU_V6) #elif defined(CONFIG_CPU_V6)
#define tls_emu 0 #define tls_emu 0
#define has_tls_reg (elf_hwcap & HWCAP_TLS) #define has_tls_reg (elf_hwcap & HWCAP_TLS)
#define defer_tls_reg_update IS_ENABLED(CONFIG_SMP) #define defer_tls_reg_update is_smp()
#define switch_tls switch_tls_v6 #define switch_tls switch_tls_v6
#elif defined(CONFIG_CPU_32v6K) #elif defined(CONFIG_CPU_32v6K)
#define tls_emu 0 #define tls_emu 0

9
arch/arm/kernel/entry-armv.S
View File

@ -38,11 +38,10 @@
#ifdef CONFIG_UNWINDER_ARM #ifdef CONFIG_UNWINDER_ARM
mov fpreg, sp @ Preserve original SP mov fpreg, sp @ Preserve original SP
#else #else
mov r8, fp @ Preserve original FP mov r7, fp @ Preserve original FP
mov r9, sp @ Preserve original SP mov r8, sp @ Preserve original SP
#endif #endif
ldr_this_cpu sp, irq_stack_ptr, r2, r3 ldr_this_cpu sp, irq_stack_ptr, r2, r3
.if \from_user == 0 .if \from_user == 0
UNWIND( .setfp fpreg, sp ) UNWIND( .setfp fpreg, sp )
@ @
@ -82,8 +81,8 @@ UNWIND( .setfp fpreg, sp )
#ifdef CONFIG_UNWINDER_ARM #ifdef CONFIG_UNWINDER_ARM
mov sp, fpreg @ Restore original SP mov sp, fpreg @ Restore original SP
#else #else
mov fp, r8 @ Restore original FP mov fp, r7 @ Restore original FP
mov sp, r9 @ Restore original SP mov sp, r8 @ Restore original SP
#endif // CONFIG_UNWINDER_ARM #endif // CONFIG_UNWINDER_ARM
#endif // CONFIG_IRQSTACKS #endif // CONFIG_IRQSTACKS
.endm .endm

17
arch/arm/kernel/entry-header.S
View File

@ -292,21 +292,18 @@
.macro restore_user_regs, fast = 0, offset = 0 .macro restore_user_regs, fast = 0, offset = 0
#if defined(CONFIG_CPU_32v6K) || defined(CONFIG_SMP) #if defined(CONFIG_CPU_32v6K) && \
#if defined(CONFIG_CPU_V6) && defined(CONFIG_SMP) (!defined(CONFIG_CPU_V6) || defined(CONFIG_SMP))
ALT_SMP(b .L1_\@ ) #ifdef CONFIG_CPU_V6
ALT_UP( nop ) ALT_SMP(nop)
ldr_va r1, elf_hwcap ALT_UP_B(.L1_\@)
tst r1, #HWCAP_TLS @ hardware TLS available?
beq .L2_\@
.L1_\@:
#endif #endif
@ The TLS register update is deferred until return to user space so we @ The TLS register update is deferred until return to user space so we
@ can use it for other things while running in the kernel @ can use it for other things while running in the kernel
get_thread_info r1 mrc p15, 0, r1, c13, c0, 3 @ get current_thread_info pointer
ldr r1, [r1, #TI_TP_VALUE] ldr r1, [r1, #TI_TP_VALUE]
mcr p15, 0, r1, c13, c0, 3 @ set TLS register mcr p15, 0, r1, c13, c0, 3 @ set TLS register
.L2_\@: .L1_\@:
#endif #endif
uaccess_enable r1, isb=0 uaccess_enable r1, isb=0

7
arch/arm/kernel/head.S
View File

@ -424,6 +424,13 @@ ENDPROC(secondary_startup)
ENDPROC(secondary_startup_arm) ENDPROC(secondary_startup_arm)
ENTRY(__secondary_switched) ENTRY(__secondary_switched)
#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE)
@ Before using the vmap'ed stack, we have to switch to swapper_pg_dir
@ as the ID map does not cover the vmalloc region.
mrc p15, 0, ip, c2, c0, 1 @ read TTBR1
mcr p15, 0, ip, c2, c0, 0 @ set TTBR0
instr_sync
#endif
adr_l r7, secondary_data + 12 @ get secondary_data.stack adr_l r7, secondary_data + 12 @ get secondary_data.stack
ldr sp, [r7] ldr sp, [r7]
ldr r0, [r7, #4] @ get secondary_data.task ldr r0, [r7, #4] @ get secondary_data.task

7
arch/arm/kernel/module.c
View File

@ -68,6 +68,7 @@ bool module_exit_section(const char *name)
strstarts(name, ".ARM.exidx.exit"); strstarts(name, ".ARM.exidx.exit");
} }
#ifdef CONFIG_ARM_HAS_GROUP_RELOCS
/* /*
* This implements the partitioning algorithm for group relocations as * This implements the partitioning algorithm for group relocations as
* documented in the ARM AArch32 ELF psABI (IHI 0044). * documented in the ARM AArch32 ELF psABI (IHI 0044).
@ -103,6 +104,7 @@ static u32 get_group_rem(u32 group, u32 *offset)
} while (group--); } while (group--);
return shift; return shift;
} }
#endif
int int
apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
@ -118,7 +120,9 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
unsigned long loc; unsigned long loc;
Elf32_Sym *sym; Elf32_Sym *sym;
const char *symname; const char *symname;
#ifdef CONFIG_ARM_HAS_GROUP_RELOCS
u32 shift, group = 1; u32 shift, group = 1;
#endif
s32 offset; s32 offset;
u32 tmp; u32 tmp;
#ifdef CONFIG_THUMB2_KERNEL #ifdef CONFIG_THUMB2_KERNEL
@ -249,6 +253,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
*(u32 *)loc = __opcode_to_mem_arm(tmp); *(u32 *)loc = __opcode_to_mem_arm(tmp);
break; break;
#ifdef CONFIG_ARM_HAS_GROUP_RELOCS
case R_ARM_ALU_PC_G0_NC: case R_ARM_ALU_PC_G0_NC:
group = 0; group = 0;
fallthrough; fallthrough;
@ -296,7 +301,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
} }
*(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset);
break; break;
#endif
#ifdef CONFIG_THUMB2_KERNEL #ifdef CONFIG_THUMB2_KERNEL
case R_ARM_THM_CALL: case R_ARM_THM_CALL:
case R_ARM_THM_JUMP24: case R_ARM_THM_JUMP24:

7
arch/arm/kernel/sleep.S
View File

@ -119,6 +119,13 @@ ENTRY(cpu_resume_mmu)
ENDPROC(cpu_resume_mmu) ENDPROC(cpu_resume_mmu)
.popsection .popsection
cpu_resume_after_mmu: cpu_resume_after_mmu:
#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE)
@ Before using the vmap'ed stack, we have to switch to swapper_pg_dir
@ as the ID map does not cover the vmalloc region.
mrc p15, 0, ip, c2, c0, 1 @ read TTBR1
mcr p15, 0, ip, c2, c0, 0 @ set TTBR0
instr_sync
#endif
bl cpu_init @ restore the und/abt/irq banked regs bl cpu_init @ restore the und/abt/irq banked regs
mov r0, #0 @ return zero on success mov r0, #0 @ return zero on success
ldmfd sp!, {r4 - r11, pc} ldmfd sp!, {r4 - r11, pc}

5
arch/arm/kernel/smp.c
View File

@ -405,11 +405,6 @@ static void smp_store_cpu_info(unsigned int cpuid)
static void set_current(struct task_struct *cur) static void set_current(struct task_struct *cur)
{ {
if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && !is_smp()) {
__current = cur;
return;
}
/* Set TPIDRURO */ /* Set TPIDRURO */
asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");
} }

25
arch/arm/kernel/traps.c
View File

@ -885,6 +885,7 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs)
die("kernel stack overflow", regs, 0); die("kernel stack overflow", regs, 0);
} }
#ifndef CONFIG_ARM_LPAE
/* /*
* Normally, we rely on the logic in do_translation_fault() to update stale PMD * Normally, we rely on the logic in do_translation_fault() to update stale PMD
* entries covering the vmalloc space in a task's page tables when it first * entries covering the vmalloc space in a task's page tables when it first
@ -895,26 +896,14 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs)
* So we need to ensure that these PMD entries are up to date *before* the MM * So we need to ensure that these PMD entries are up to date *before* the MM
* switch. As we already have some logic in the MM switch path that takes care * switch. As we already have some logic in the MM switch path that takes care
* of this, let's trigger it by bumping the counter every time the core vmalloc * of this, let's trigger it by bumping the counter every time the core vmalloc
* code modifies a PMD entry in the vmalloc region. * code modifies a PMD entry in the vmalloc region. Use release semantics on
* the store so that other CPUs observing the counter's new value are
* guaranteed to see the updated page table entries as well.
*/ */
void arch_sync_kernel_mappings(unsigned long start, unsigned long end) void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
{ {
if (start > VMALLOC_END || end < VMALLOC_START) if (start < VMALLOC_END && end > VMALLOC_START)
return; atomic_inc_return_release(&init_mm.context.vmalloc_seq);
/*
* This hooks into the core vmalloc code to receive notifications of
* any PMD level changes that have been made to the kernel page tables.
* This means it should only be triggered once for every MiB worth of
* vmalloc space, given that we don't support huge vmalloc/vmap on ARM,
* and that kernel PMD level table entries are rarely (if ever)
* updated.
*
* This means that the counter is going to max out at ~250 for the
* typical case. If it overflows, something entirely unexpected has
* occurred so let's throw a warning if that happens.
*/
WARN_ON(++init_mm.context.vmalloc_seq == UINT_MAX);
} }
#endif
#endif #endif

2
arch/arm/mach-iop32x/irq.c
View File

@ -59,7 +59,7 @@ struct irq_chip ext_chip = {
.irq_unmask = iop32x_irq_unmask, .irq_unmask = iop32x_irq_unmask,
}; };
void iop_handle_irq(struct pt_regs *regs) static void iop_handle_irq(struct pt_regs *regs)
{ {
u32 mask; u32 mask;

3
arch/arm/mm/context.c
View File

@ -240,8 +240,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
unsigned int cpu = smp_processor_id(); unsigned int cpu = smp_processor_id();
u64 asid; u64 asid;
if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) check_vmalloc_seq(mm);
__check_vmalloc_seq(mm);
/* /*
* We cannot update the pgd and the ASID atomicly with classic * We cannot update the pgd and the ASID atomicly with classic

18
arch/arm/mm/ioremap.c
View File

@ -117,16 +117,21 @@ EXPORT_SYMBOL(ioremap_page);
void __check_vmalloc_seq(struct mm_struct *mm) void __check_vmalloc_seq(struct mm_struct *mm)
{ {
unsigned int seq; int seq;
do { do {
seq = init_mm.context.vmalloc_seq; seq = atomic_read(&init_mm.context.vmalloc_seq);
memcpy(pgd_offset(mm, VMALLOC_START), memcpy(pgd_offset(mm, VMALLOC_START),
pgd_offset_k(VMALLOC_START), pgd_offset_k(VMALLOC_START),
sizeof(pgd_t) * (pgd_index(VMALLOC_END) - sizeof(pgd_t) * (pgd_index(VMALLOC_END) -
pgd_index(VMALLOC_START))); pgd_index(VMALLOC_START)));
mm->context.vmalloc_seq = seq; /*
} while (seq != init_mm.context.vmalloc_seq); * Use a store-release so that other CPUs that observe the
* counter's new value are guaranteed to see the results of the
* memcpy as well.
*/
atomic_set_release(&mm->context.vmalloc_seq, seq);
} while (seq != atomic_read(&init_mm.context.vmalloc_seq));
} }
#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
@ -157,7 +162,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size)
* Note: this is still racy on SMP machines. * Note: this is still racy on SMP machines.
*/ */
pmd_clear(pmdp); pmd_clear(pmdp);
init_mm.context.vmalloc_seq++; atomic_inc_return_release(&init_mm.context.vmalloc_seq);
/* /*
* Free the page table, if there was one. * Free the page table, if there was one.
@ -174,8 +179,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size)
* Ensure that the active_mm is up to date - we want to * Ensure that the active_mm is up to date - we want to
* catch any use-after-iounmap cases. * catch any use-after-iounmap cases.
*/ */
if (current->active_mm->context.vmalloc_seq != init_mm.context.vmalloc_seq) check_vmalloc_seq(current->active_mm);
__check_vmalloc_seq(current->active_mm);
flush_tlb_kernel_range(virt, end); flush_tlb_kernel_range(virt, end);
} }